From 8d774ab6d3734432b4aff4855645cb24f9fabee3 Mon Sep 17 00:00:00 2001
From: missinglink <insomnia@rcpt.at>
Date: Mon, 3 Apr 2017 14:56:02 +0200
Subject: [PATCH] language service

---
 middleware/changeLanguage.js           | 172 ++++++++++++++++++++
 routes/v1.js                           |   9 +-
 service/language.js                    |  93 +++++++++++
 test/unit/middleware/changeLanguage.js | 216 +++++++++++++++++++++++++
 test/unit/run.js                       |   4 +-
 test/unit/service/language.js          | 107 ++++++++++++
 6 files changed, 599 insertions(+), 2 deletions(-)
 create mode 100644 middleware/changeLanguage.js
 create mode 100644 service/language.js
 create mode 100644 test/unit/middleware/changeLanguage.js
 create mode 100644 test/unit/service/language.js

diff --git a/middleware/changeLanguage.js b/middleware/changeLanguage.js
new file mode 100644
index 00000000..d6449ae8
--- /dev/null
+++ b/middleware/changeLanguage.js
@@ -0,0 +1,172 @@
+
+var logger = require( 'pelias-logger' ).get( 'api' );
+var service = require('../service/language');
+
+/**
+example response from language web service:
+{
+  "101748479": {
+    "wofid": 101748479,
+    "placetype": "locality",
+    "iso": "DE",
+    "area": 0.031614,
+    "lineage": {
+      "continent_id": 102191581,
+      "country_id": 85633111,
+      "county_id": 102063261,
+      "locality_id": 101748479,
+      "macrocounty_id": 404227567,
+      "region_id": 85682571
+    },
+    "rowid": 90425,
+    "names": {
+      "default": "München",
+      "eng": "Munich"
+    }
+  },
+}
+**/
+
+function setup() {
+
+  var transport = service.findById();
+  var middleware = function(req, res, next) {
+
+    // no-op, request did not require a language change
+    if( !isLanguageChangeRequired( req, res ) ){
+      return next();
+    }
+
+    // collect a list of parent ids to fetch translations for
+    var ids = extractIds( res );
+
+    // perform language lookup for all relevant ids
+    var timer = (new Date()).getTime();
+    transport.query( ids, function( err, translations ){
+
+      // update documents using a translation map
+      if( err ){
+        logger.error( '[language] [error]', err );
+      } else {
+        updateDocs( req, res, translations );
+      }
+
+      logger.info( '[language] [took]', (new Date()).getTime() - timer, 'ms' );
+      next();
+    });
+  };
+
+  middleware.transport = transport;
+  return middleware;
+}
+
+// collect a list of parent ids to fetch translations for
+function extractIds( res ){
+
+  // store ids in an object in order to avoid duplicates
+  var ids = {};
+
+  // convenience function for adding a new id to the object
+  function addId(id) {
+    ids[id] = true;
+  }
+
+  // extract all parent ids from documents
+  res.data.forEach( function( doc ){
+
+    // skip invalid records
+    if( !doc || !doc.parent ){ return; }
+
+    // iterate over doc.parent.* attributes
+    for( var attr in doc.parent ){
+
+      // match only attributes ending with '_id'
+      var match = attr.match(/_id$/);
+      if( !match ){ continue; }
+
+      // skip invalid/empty arrays
+      if( !Array.isArray( doc.parent[attr] ) || !doc.parent[attr].length ){
+        continue;
+      }
+
+      // add each id as a key in the ids object
+      doc.parent[attr].forEach( addId );
+    }
+  });
+
+  // return a deduplicated array of ids
+  return Object.keys( ids );
+}
+
+// update documents using a translation map
+function updateDocs( req, res, translations ){
+
+  // sanity check arguments
+  if( !req || !res || !res.data || !translations ){ return; }
+
+  // this is the target language we will be translating to
+  var requestLanguage = req.language.iso6393;
+
+  // iterate over response documents
+  res.data.forEach( function( doc, p ){
+
+    // skip invalid records
+    if( !doc || !doc.parent ){ return; }
+
+    // iterate over doc.parent.* attributes
+    for( var attr in doc.parent ){
+
+      // match only attributes ending with '_id'
+      var match = attr.match(/^(.*)_id$/);
+      if( !match ){ continue; }
+
+      // adminKey is the property name without the '_id'
+      // eg. for 'country_id', adminKey would be 'country'.
+      var adminKey = match[1];
+      var adminValues = doc.parent[adminKey];
+
+      // skip invalid/empty arrays
+      if( !Array.isArray( adminValues ) || !adminValues.length ){ continue; }
+
+      // iterate over adminValues (it's an array and can have more than one value)
+      for( var i in adminValues ){
+
+        // find the corresponding key from the '_id' Array
+        var id = doc.parent[attr][i];
+        if( !id ){ continue; }
+
+        // id not found in translation service response
+        if( !translations.hasOwnProperty( id ) ){
+          logger.error( '[language] [error]', 'failed to find translations for', id );
+          continue;
+        }
+
+        // skip invalid records
+        if( !translations[id].hasOwnProperty( 'names' ) ){ continue; }
+
+        // requested language is not available
+        if( !translations[id].names.hasOwnProperty( requestLanguage ) ){
+          logger.info( '[language] [info]', 'missing translation', requestLanguage, id );
+          continue;
+        }
+
+        // translate 'parent.*' property
+        adminValues[i] = translations[id].names[ requestLanguage ];
+
+        // if the record is an admin record we also translate
+        // the 'name.default' property.
+        if( adminKey === doc.layer ){
+          doc.name.default = translations[id].names[ requestLanguage ];
+        }
+      }
+    }
+  });
+}
+
+// boolean function to check if changing the language is required
+function isLanguageChangeRequired( req, res ){
+  return req && res && res.data && res.data.length &&
+         req.hasOwnProperty('language');
+}
+
+module.exports = setup;
diff --git a/routes/v1.js b/routes/v1.js
index 8ee4173f..c1cc4554 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -58,7 +58,8 @@ var postProc = {
   sendJSON: require('../middleware/sendJSON'),
   parseBoundingBox: require('../middleware/parseBBox'),
   normalizeParentIds: require('../middleware/normalizeParentIds'),
-  assignLabels: require('../middleware/assignLabels')
+  assignLabels: require('../middleware/assignLabels'),
+  changeLanguage: require('../middleware/changeLanguage')
 };
 
 // predicates that drive whether controller/search runs
@@ -127,6 +128,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -147,6 +149,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -163,6 +166,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -183,6 +187,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -202,6 +207,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -215,6 +221,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
+      postProc.changeLanguage(),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
diff --git a/service/language.js b/service/language.js
new file mode 100644
index 00000000..50cc29c0
--- /dev/null
+++ b/service/language.js
@@ -0,0 +1,93 @@
+
+var logger = require( 'pelias-logger' ).get( 'api' ),
+    request = require( 'superagent' ),
+    peliasConfig = require( 'pelias-config' );
+
+/**
+
+  language subsitution service client
+
+  this file provides a 'transport' which can be used to access the language
+  service via a network connnection.
+
+  the exported method for this module checks pelias-config for a configuration block such as:
+
+  "language": {
+    "client": {
+      "adapter": "http",
+      "host": "http://localhost:6100"
+    }
+  }
+
+  for more info on running the service see: https://github.com/pelias/placeholder
+
+**/
+
+/**
+  NullTransport
+
+  disables the service completely
+**/
+function NullTransport(){}
+NullTransport.prototype.query = function( coord, number, street, cb ){
+  cb(); // no-op
+};
+
+/**
+  HttpTransport
+
+  allows the api to be used via a remote web service
+**/
+function HttpTransport( host, settings ){
+  this.query = function( ids, cb ){
+    request
+      .get( host + '/parser/findbyid' )
+      .set( 'Accept', 'application/json' )
+      .query({ ids: Array.isArray( ids ) ? ids.join(',') : '' })
+      .timeout( settings && settings.timeout || 1000 )
+      .end( function( err, res ){
+        if( err || !res ){ return cb( err ); }
+        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
+        return cb( null, res.body );
+      });
+  };
+}
+HttpTransport.prototype.query = function( coord, number, street, cb ){
+  throw new Error( 'language: transport not connected' );
+};
+
+/**
+  Setup
+
+  allows instantiation of transport depending on configuration and preference
+**/
+module.exports.findById = function setup(){
+
+  // user config
+  var config = peliasConfig.generate();
+
+  // ensure config variables set correctly
+  if( !config.hasOwnProperty('language') || !config.language.hasOwnProperty('client') ){
+    logger.warn( 'language: configuration not found' );
+  }
+
+  // valid configuration found
+  else {
+
+    // get adapter settings from config
+    var settings = config.language.client;
+
+    // http adapter
+    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
+      logger.info( 'language: using http transport:', settings.host );
+      if( settings.hasOwnProperty('timeout') ){
+        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
+      }
+      return new HttpTransport( settings.host );
+    }
+  }
+
+  // default adapter
+  logger.info( 'language: using null transport' );
+  return new NullTransport();
+};
diff --git a/test/unit/middleware/changeLanguage.js b/test/unit/middleware/changeLanguage.js
new file mode 100644
index 00000000..66b6d127
--- /dev/null
+++ b/test/unit/middleware/changeLanguage.js
@@ -0,0 +1,216 @@
+
+var fs = require('fs'),
+    tmp = require('tmp'),
+    setup = require('../../../middleware/changeLanguage');
+
+// load middleware using the default pelias config
+var load = function(){
+  // adapter is driven by config
+  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
+  process.env.PELIAS_CONFIG = tmpfile;
+  var middleware = setup();
+  delete process.env.PELIAS_CONFIG;
+  return middleware;
+};
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    var middleware = load();
+    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(middleware.length, 3, 'middleware is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.isLanguageChangeRequired = function(test, common) {
+  test('invalid query - null req/res', function(t) {
+    var middleware = load();
+    middleware(null, null, t.end);
+  });
+
+  test('invalid query - no results', function(t) {
+    var req = { language: { iso6393: 'spa' } };
+    var res = {};
+
+    var middleware = load();
+    middleware(req, res, function(){
+      t.deepEqual( req, { language: { iso6393: 'spa' } } );
+      t.deepEqual( res, {} );
+      t.end();
+    });
+  });
+
+  test('invalid query - empty results', function(t) {
+    var req = { language: { iso6393: 'spa' } };
+    var res = { data: [] };
+
+    var middleware = load();
+    middleware(req, res, function(){
+      t.deepEqual( req, { language: { iso6393: 'spa' } } );
+      t.deepEqual( res, { data: [] } );
+      t.end();
+    });
+  });
+
+  test('invalid query - no target language', function(t) {
+    var req = {};
+    var res = { data: [] };
+
+    var middleware = load();
+    middleware(req, res, function(){
+      t.deepEqual( req, {} );
+      t.deepEqual( res, { data: [] } );
+      t.end();
+    });
+  });
+};
+
+// check the service is called and response mapped correctly
+module.exports.tests.miss = function(test, common) {
+  test('miss', function(t) {
+
+    var req = { language: { iso6393: 'spa' } };
+    var res = { data: [
+      {
+        layer: 'locality',
+        name: { default: 'London' },
+        parent: {
+          locality_id: [ 101750367 ],
+          locality: [ 'London' ]
+        }
+      },
+      {
+        layer: 'example',
+        name: { default: 'London' },
+        parent: {
+          locality_id: [ 101735809 ],
+          locaity: [ 'London' ]
+        }
+      }
+    ]};
+
+    var middleware = load();
+
+    // mock out the transport
+    middleware.transport.query = function mock( ids, cb ){
+      t.deepEqual( ids, [ '101735809', '101750367' ] );
+      t.equal( typeof cb, 'function' );
+      cb( 'error' );
+    };
+
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [
+        {
+          layer: 'locality',
+          name: { default: 'London' },
+          parent: {
+            locality_id: [ 101750367 ],
+            locality: [ 'London' ]
+          }
+        },
+        {
+          layer: 'example',
+          name: { default: 'London' },
+          parent: {
+            locality_id: [ 101735809 ],
+            locaity: [ 'London' ]
+          }
+        }
+      ]});
+      t.end();
+    });
+  });
+};
+
+// check the service is called and response mapped correctly
+module.exports.tests.hit = function(test, common) {
+  test('hit', function(t) {
+
+    var req = { language: { iso6393: 'spa' } };
+    var res = { data: [
+      {
+        layer: 'locality',
+        name: { default: 'London' },
+        parent: {
+          locality_id: [ 101750367 ],
+          locality: [ 'London' ]
+        }
+      },
+      {
+        layer: 'example',
+        name: { default: 'London' },
+        parent: {
+          locality_id: [ 101735809 ],
+          locaity: [ 'London' ]
+        }
+      }
+    ]};
+
+    var middleware = load();
+
+    // mock out the transport
+    middleware.transport.query = function mock( ids, cb ){
+      t.deepEqual( ids, [ '101735809', '101750367' ] );
+      t.equal( typeof cb, 'function' );
+      cb( null, {
+        '101750367': {
+          'names': {
+            'default':'London',
+            'chi':'倫敦',
+            'spa':'Londres',
+            'eng':'London',
+            'hin':'लंदन',
+            'ara':'لندن',
+            'por':'Londres',
+            'ben':'লন্ডন',
+            'rus':'Лондон',
+            'jpn':'ロンドン',
+            'kor':'런던'
+          }
+        },
+        '101735809': {
+          'names':{
+            'default':'London',
+            'eng':'London'
+          }
+        }
+      });
+    };
+
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [
+        {
+          layer: 'locality',
+          name: { default: 'Londres' },
+          parent: {
+            locality_id: [ 101750367 ],
+            locality: [ 'Londres' ]
+          }
+        },
+        {
+          layer: 'example',
+          name: { default: 'London' },
+          parent: {
+            locality_id: [ 101735809 ],
+            locaity: [ 'London' ]
+          }
+        }
+      ]});
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[middleware] changeLanguage: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/test/unit/run.js b/test/unit/run.js
index ad358e30..3c7d01e4 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -30,6 +30,7 @@ var tests = [
   require('./middleware/confidenceScore'),
   require('./middleware/confidenceScoreFallback'),
   require('./middleware/confidenceScoreReverse'),
+  require('./middleware/changeLanguage'),
   require('./middleware/distance'),
   require('./middleware/interpolate'),
   require('./middleware/localNamingConventions'),
@@ -80,7 +81,8 @@ var tests = [
   require('./service/mget'),
   require('./service/search'),
   require('./service/interpolation'),
-  require('./service/pointinpolygon')
+  require('./service/pointinpolygon'),
+  require('./service/language')
 ];
 
 tests.map(function(t) {
diff --git a/test/unit/service/language.js b/test/unit/service/language.js
new file mode 100644
index 00000000..151247fd
--- /dev/null
+++ b/test/unit/service/language.js
@@ -0,0 +1,107 @@
+
+var fs = require('fs'),
+    tmp = require('tmp'),
+    setup = require('../../../service/language').findById;
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof setup, 'function', 'setup is a function');
+    t.end();
+  });
+};
+
+// adapter factory
+module.exports.tests.factory = function(test, common) {
+
+  test('http adapter', function(t) {
+    var config = { language: { client: {
+      adapter: 'http',
+      host: 'http://example.com'
+    }}};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'HttpTransport', 'HttpTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 2, 'query function signature');
+    t.end();
+  });
+
+  test('null adapter', function(t) {
+    var config = { language: { client: {
+      adapter: 'null'
+    }}};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+  test('default adapter', function(t) {
+    var config = {};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+};
+
+// null transport
+module.exports.tests.NullTransport = function(test, common) {
+
+  test('null transport', function(t) {
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    // test null transport performs a no-op
+    adapter.query( null, null, null, function( err, res ){
+      t.equal(err, undefined, 'no-op');
+      t.equal(res, undefined, 'no-op');
+      t.end();
+    });
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SERVICE language', testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};