Merge pull request #936 from pelias/convert-language-and-interpolation-to-microservices

Convert language and interpolation to microservices

controller/predicates/has_any_parsed_text_property.js

@@ -1,18 +0,0 @@
-const _ = require('lodash');
-
-// return true if any setup parameter is a key of request.clean.parsed_text
-// "arguments" is only available in long-form function declarations, cannot be shortened to fat arrow syntax
-// potential improvement:  inject set operator to allow for any/all functionality
-module.exports = function() {
-  // save off requested properties since arguments can't be referenced later
-  const properties = _.values(arguments);
-
-  // return true if any of the supplied properties are in clean.parsed_text
-  return (request, response) => !_.isEmpty(
-    _.intersection(
-      properties,
-      _.keys(_.get(request, ['clean', 'parsed_text'], {}))
-    )
-  );
-
-};

controller/predicates/has_parsed_text_properties.js

@@ -0,0 +1,33 @@
+const _ = require('lodash');
+
+// "arguments" is only available in long-form function declarations, cannot be shortened to fat arrow syntax
+// potential improvement:  inject set operator to allow for any/all functionality
+module.exports = {
+  all: function() {
+    // save off property names for future reference
+    const properties = _.values(arguments);
+
+    // return true if ALL of the supplied properties are in clean.parsed_text
+    return request => _.isEmpty(
+      _.difference(
+        _.values(properties),
+        _.keys(_.get(request, ['clean', 'parsed_text'], {}))
+      )
+    );
+
+  },
+  any: function() {
+    // save off property names for future reference
+    const properties = _.values(arguments);
+
+    // return true if ANY of the supplied properties are in clean.parsed_text
+    return request => !_.isEmpty(
+      _.intersection(
+        _.values(properties),
+        _.keys(_.get(request, ['clean', 'parsed_text'], {}))
+      )
+    );
+
+  }
+
+};

middleware/changeLanguage.js

@@ -1,6 +1,5 @@
 
 var logger = require( 'pelias-logger' ).get( 'api' );
-var service = require('../service/language');
 const _ = require('lodash');
 
 /**
@@ -28,84 +27,34 @@ example response from language web service:
 }
 **/
 
-function setup() {
+function setup(service, should_execute) {
-  var transport = service.findById();
+  return function controller(req, res, next) {
-  var middleware = function(req, res, next) {
+    if (!should_execute(req, res)) {
-
-    // no-op, request did not require a language change
-    if( !isLanguageChangeRequired( req, res ) ){
       return next();
     }
 
-    // collect a list of parent ids to fetch translations for
+    service(req, res, (err, translations) => {
-    var ids = extractIds( res );
+      // if there's an error, log it and bail
-
+      if (err) {
-    // perform language lookup for all relevant ids
+        logger.info(`[middleware:language][error]`);
-    var timer = (new Date()).getTime();
+        logger.error(err);
-    transport.query( ids, function( err, translations ){
+        return next();
-
-      // update documents using a translation map
-      if( err ){
-        logger.error( '[language] [error]', err );
-      } else {
-        updateDocs( req, res, translations );
       }
 
-      logger.info( '[language] [took]', (new Date()).getTime() - timer, 'ms' );
+      // otherwise, update all the docs with translations
+      updateDocs(req, res, _.defaultTo(translations, []));
       next();
-    });
-  };
-
-  middleware.transport = transport;
-  return middleware;
-}
-
-// collect a list of parent ids to fetch translations for
-function extractIds( res ){
 
-  // store ids in an object in order to avoid duplicates
-  var ids = {};
-
-  // convenience function for adding a new id to the object
-  function addId(id) {
-    ids[id] = true;
-  }
-
-  // extract all parent ids from documents
-  res.data.forEach( function( doc ){
-
-    // skip invalid records
-    if( !doc || !doc.parent ){ return; }
-
-    // iterate over doc.parent.* attributes
-    for( var attr in doc.parent ){
-
-      // match only attributes ending with '_id'
-      var match = attr.match(/_id$/);
-      if( !match ){ continue; }
-
-      // skip invalid/empty arrays
-      if( !Array.isArray( doc.parent[attr] ) || !doc.parent[attr].length ){
-        continue;
-      }
-
-      // add each id as a key in the ids object
-      doc.parent[attr].forEach( addId );
-    }
     });
 
-  // return a deduplicated array of ids
+  };
-  return Object.keys( ids );
+
 }
 
 // update documents using a translation map
 function updateDocs( req, res, translations ){
-
-  // sanity check arguments
-  if( !req || !res || !res.data || !translations ){ return; }
-
   // this is the target language we will be translating to
-  var requestLanguage = req.language.iso6393;
+  var requestLanguage = req.clean.lang.iso6393;
 
   // iterate over response documents
   res.data.forEach( function( doc, p ){
@@ -136,17 +85,14 @@ function updateDocs( req, res, translations ){
         if( !id ){ continue; }
 
         // id not found in translation service response
-        if( !translations.hasOwnProperty( id ) ){
+        if( !_.has(translations, id)){
-          logger.error( '[language] [error]', 'failed to find translations for', id );
+          logger.debug( `[language] [debug] failed to find translations for ${id}` );
           continue;
         }
 
-        // skip invalid records
-        if( !translations[id].hasOwnProperty( 'names' ) ){ continue; }
-
         // requested language is not available
         if (_.isEmpty(_.get(translations[id].names, requestLanguage, [] ))) {
-          logger.debug( '[language] [debug]', 'missing translation', requestLanguage, id );
+          logger.debug( `[language] [debug] missing translation ${requestLanguage} ${id}` );
           continue;
         }
 

middleware/interpolate.js

@@ -1,7 +1,7 @@
-
+const async = require('async');
-var async = require('async');
+const logger = require( 'pelias-logger' ).get( 'api' );
-var logger = require( 'pelias-logger' ).get( 'api' );
+const source_mapping = require('../helper/type_mapping').source_mapping;
-var service = require('../service/interpolation');
+const _ = require('lodash');
 
 /**
 example response from interpolation web service:
@@ -21,132 +21,96 @@ example response from interpolation web service:
 }
 **/
 
-function setup() {
+function setup(service, should_execute) {
-
+  return function controller(req, res, next) {
-  var transport = service.search();
+    if (!should_execute(req, res)) {
-  var middleware = function(req, res, next) {
-
-    // no-op, user did not request an address
-    if( !isAddressQuery( req ) ){
       return next();
     }
 
-    // bind parsed_text variables to function call
+    // bind the service to the req which doesn't change
-    var bound = interpolate.bind( transport, req.clean.parsed_text );
+    const req_bound_service = _.partial(service, req);
-
-    // perform interpolations asynchronously for all relevant hits
-    var timer = (new Date()).getTime();
-    async.map( res.data, bound, function( err, results ){
-
-      // update res.data with the mapped values
-      if( !err ){
-        res.data = results;
-      }
-
-      // sort the results to ensure that addresses show up higher than street centroids      
-      res.data = res.data.sort((a, b) => {
-        if (a.layer === 'address' && b.layer !== 'address') { return -1; }
-        if (a.layer !== 'address' && b.layer === 'address') { return 1; }
-        return 0;
-      });
-
-      // log the execution time, continue
-      logger.info( '[interpolation] [took]', (new Date()).getTime() - timer, 'ms' );
-      next();
-    });
-  };
-
-  middleware.transport = transport;
-  return middleware;
-}
 
-function interpolate( parsed_text, hit, cb ){
+    // only interpolate the street-layer results
+    // save this off into a separate array so that when docs are annotated
+    //  after the interpolate results are returned, no complicated bookkeeping is needed
+    const street_results = _.get(res, 'data', []).filter(result => result.layer === 'street');
 
-  // no-op, this hit is not from the 'street' layer
+    // perform interpolations asynchronously for all relevant hits
-  // note: no network request is performed.
+    const start = (new Date()).getTime();
-  if( !hit || hit.layer !== 'street' ){
+    async.map(street_results, req_bound_service, (err, interpolation_results) => {
-    return cb( null, hit );
+      if (err) {
+        logger.error(`[middleware:interpolation] ${_.defaultTo(err.message, err)}`);
+        return next();
       }
 
-  // query variables
+      interpolation_results.forEach((interpolation_result, idx) => {
-  var coord = hit.center_point;
+        const source_result = street_results[idx];
-  var number = parsed_text.number;
-  var street = hit.address_parts.street || parsed_text.street;
-
-  // query interpolation service
-  this.query( coord, number, street, function( err, data ){
-
-    // an error occurred
-    // note: leave this hit unmodified
-    if( err ){
-      logger.error( '[interpolation] [error]', err );
-      return cb( null, hit );
-    }
 
-    // invalid / not useful response
+        // invalid / not useful response, debug log for posterity
         // note: leave this hit unmodified
-    if( !data || !data.hasOwnProperty('properties') ){
+        if (!_.has(interpolation_result, 'properties')) {
-      logger.info( '[interpolation] [miss]', parsed_text );
+          logger.debug(`[interpolation] [miss] ${req.clean.parsed_text}`);
-      return cb( null, hit );
+          return;
         }
 
-    // the interpolation service returned a valid result
+        // the interpolation service returned a valid result, debug log for posterity
-    // note: we now merge thos values with the existing 'street' record.
+        // note: we now merge those values with the existing 'street' record
-    logger.info( '[interpolation] [hit]', parsed_text, data );
+        logger.debug(`[interpolation] [hit] ${req.clean.parsed_text} ${JSON.stringify(interpolation_result)}`);
-
-    // safety first!
-    try {
 
-      // -- metatdata --
+        // -- metadata --
-      hit.layer = 'address';
+        source_result.layer = 'address';
-      hit.match_type = 'interpolated';
+        source_result.match_type = 'interpolated';
 
         // -- name --
-      hit.name.default = data.properties.number + ' ' + hit.name.default;
+        source_result.name.default = `${interpolation_result.properties.number} ${source_result.name.default}`;
 
         // -- source --
-      var source = 'mixed';
+        // lookup the lowercased source, defaulting to 'mixed' when not found
-      if( data.properties.source === 'OSM' ){ source = 'openstreetmap'; }
+        // the source mapping is a jagged string->array, so default to 'mixed' as an array
-      else if( data.properties.source === 'OA' ){ source = 'openaddresses'; }
+        //  to ensure that subscript works
-      hit.source = source;
+        source_result.source = _.defaultTo(
+          source_mapping[_.toLower(interpolation_result.properties.source)],
+          ['mixed']
+        )[0];
 
         // -- source_id --
         // note: interpolated values have no source_id
-      delete hit.source_id; // remove original street source_id
+        delete source_result.source_id; // remove original street source_id
-      if( data.properties.hasOwnProperty( 'source_id' ) ){
+        if( interpolation_result.properties.hasOwnProperty( 'source_id' ) ){
-        hit.source_id = data.properties.source_id;
+          source_result.source_id = interpolation_result.properties.source_id;
         }
 
         // -- address_parts --
-      hit.address_parts.number = data.properties.number;
+        source_result.address_parts.number = interpolation_result.properties.number;
 
         // -- geo --
-      hit.center_point = {
+        source_result.center_point = {
-        lat: data.properties.lat,
+          lat: interpolation_result.properties.lat,
-        lon: data.properties.lon
+          lon: interpolation_result.properties.lon
         };
 
         // -- bbox --
-      delete hit.bounding_box;
+        delete source_result.bounding_box;
 
-      // return the modified hit
+      });
-      return cb( null, hit );
 
-    // a syntax error occurred in the code above (this shouldn't happen!)
+      // sort the results to ensure that addresses show up higher than street centroids
-    // note: the hit object may be partially modified, could possibly be invalid
+      if (_.has(res, 'data')) {
-    } catch( e ){
+        res.data.sort((a, b) => {
-      logger.error( '[interpolation] [error]', e, e.stack );
+          if (a.layer === 'address' && b.layer !== 'address') { return -1; }
-      return cb( null, hit );
+          if (a.layer !== 'address' && b.layer === 'address') { return 1; }
+          return 0;
+        });
       }
+
+      // log the execution time, continue
+      logger.info( `[interpolation] [took] ${(new Date()).getTime() - start} ms`);
+      next();
+
     });
-}
 
-// boolean function to check if an address was requested by the user
+  };
-function isAddressQuery( req ){
+
-  return req && req.hasOwnProperty('clean') &&
-         req.clean.hasOwnProperty('parsed_text') &&
-         req.clean.parsed_text.hasOwnProperty('number') &&
-         req.clean.parsed_text.hasOwnProperty('street');
 }
 
 module.exports = setup;

package.json

@@ -66,7 +66,6 @@
     "predicates": "^1.0.1",
     "retry": "^0.10.1",
     "stats-lite": "^2.0.4",
-    "superagent": "^3.2.1",
     "through2": "^2.0.3"
   },
   "devDependencies": {

routes/v1.js

@@ -69,7 +69,6 @@ var postProc = {
 };
 
 // predicates that drive whether controller/search runs
-const hasAnyParsedTextProperty = require('../controller/predicates/has_any_parsed_text_property');
 const hasResponseData = require('../controller/predicates/has_response_data');
 const hasRequestErrors = require('../controller/predicates/has_request_errors');
 const isCoarseReverse = require('../controller/predicates/is_coarse_reverse');
@@ -80,19 +79,23 @@ const hasRequestCategories = require('../controller/predicates/has_request_param
 const isOnlyNonAdminLayers = require('../controller/predicates/is_only_non_admin_layers');
 // this can probably be more generalized
 const isRequestSourcesOnlyWhosOnFirst = require('../controller/predicates/is_request_sources_only_whosonfirst');
+const hasRequestParameter = require('../controller/predicates/has_request_parameter');
+const hasParsedTextProperties = require('../controller/predicates/has_parsed_text_properties');
 
 // shorthand for standard early-exit conditions
 const hasResponseDataOrRequestErrors = any(hasResponseData, hasRequestErrors);
 const hasAdminOnlyResults = not(hasResultsAtLayers(['venue', 'address', 'street']));
 
 const hasNumberButNotStreet = all(
-  hasAnyParsedTextProperty('number'),
+  hasParsedTextProperties.any('number'),
-  not(hasAnyParsedTextProperty('street'))
+  not(hasParsedTextProperties.any('street'))
 );
 
 const serviceWrapper = require('pelias-microservice-wrapper').service;
 const PlaceHolder = require('../service/configurations/PlaceHolder');
 const PointInPolygon = require('../service/configurations/PointInPolygon');
+const Language = require('../service/configurations/Language');
+const Interpolation = require('../service/configurations/Interpolation');
 
 /**
  * Append routes to app
@@ -111,6 +114,14 @@ function addRoutes(app, peliasConfig) {
   const placeholderService = serviceWrapper(placeholderConfiguration);
   const isPlaceholderServiceEnabled = _.constant(placeholderConfiguration.isEnabled());
 
+  const changeLanguageConfiguration = new Language(_.defaultTo(peliasConfig.api.services.placeholder, {}));
+  const changeLanguageService = serviceWrapper(changeLanguageConfiguration);
+  const isChangeLanguageEnabled = _.constant(changeLanguageConfiguration.isEnabled());
+
+  const interpolationConfiguration = new Interpolation(_.defaultTo(peliasConfig.api.services.interpolation, {}));
+  const interpolationService = serviceWrapper(interpolationConfiguration);
+  const isInterpolationEnabled = _.constant(interpolationConfiguration.isEnabled());
+
   // fallback to coarse reverse when regular reverse didn't return anything
   const coarseReverseShouldExecute = all(
     isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData)
@@ -150,20 +161,20 @@ function addRoutes(app, peliasConfig) {
     // check clean.parsed_text for several conditions that must all be true
     all(
       // run placeholder if clean.parsed_text has 'street'
-      hasAnyParsedTextProperty('street'),
+      hasParsedTextProperties.any('street'),
       // don't run placeholder if there's a query or category
-      not(hasAnyParsedTextProperty('query', 'category')),
+      not(hasParsedTextProperties.any('query', 'category')),
       // run placeholder if there are any adminareas identified
-      hasAnyParsedTextProperty('neighbourhood', 'borough', 'city', 'county', 'state', 'country')
+      hasParsedTextProperties.any('neighbourhood', 'borough', 'city', 'county', 'state', 'country')
     )
   );
 
   const searchWithIdsShouldExecute = all(
     not(hasRequestErrors),
     // don't search-with-ids if there's a query or category
-    not(hasAnyParsedTextProperty('query', 'category')),
+    not(hasParsedTextProperties.any('query', 'category')),
     // there must be a street
-    hasAnyParsedTextProperty('street')
+    hasParsedTextProperties.any('street')
   );
 
   // placeholder should have executed, useful for determining whether to actually
@@ -195,6 +206,26 @@ function addRoutes(app, peliasConfig) {
     isAddressItParse
   );
 
+  // get language adjustments if:
+  // - there's a response
+  // - theres's a lang parameter in req.clean
+  const changeLanguageShouldExecute = all(
+    hasResponseData,
+    not(hasRequestErrors),
+    isChangeLanguageEnabled,
+    hasRequestParameter('lang')
+  );
+
+  // interpolate if:
+  // - there's a number and street
+  // - there are street-layer results (these are results that need to be interpolated)
+  const interpolationShouldExecute = all(
+    not(hasRequestErrors),
+    isInterpolationEnabled,
+    hasParsedTextProperties.all('number', 'street'),
+    hasResultsAtLayers('street')
+  );
+
   // execute under the following conditions:
   // - there are no errors or data
   // - request is not coarse OR pip service is disabled
@@ -238,7 +269,7 @@ function addRoutes(app, peliasConfig) {
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig.api),
       postProc.confidenceScoresFallback(),
-      postProc.interpolate(),
+      postProc.interpolate(interpolationService, interpolationShouldExecute),
       postProc.sortResponseData(require('pelias-sorting'), hasAdminOnlyResults),
       postProc.dedupe(),
       postProc.accuracy(),
@@ -246,7 +277,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -260,14 +291,14 @@ function addRoutes(app, peliasConfig) {
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig.api),
       postProc.confidenceScoresFallback(),
-      postProc.interpolate(),
+      postProc.interpolate(interpolationService, interpolationShouldExecute),
       postProc.dedupe(),
       postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -284,7 +315,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -305,7 +336,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -325,7 +356,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -339,7 +370,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON

schema.js

@@ -36,6 +36,11 @@ module.exports = Joi.object().keys({
         url: Joi.string().uri({ scheme: /https?/ }),
         timeout: Joi.number().integer().optional().default(250).min(0),
         retries: Joi.number().integer().optional().default(3).min(0),
+      }).unknown(false).requiredKeys('url'),
+      interpolation: Joi.object().keys({
+        url: Joi.string().uri({ scheme: /https?/ }),
+        timeout: Joi.number().integer().optional().default(250).min(0),
+        retries: Joi.number().integer().optional().default(3).min(0),
       }).unknown(false).requiredKeys('url')
     }).unknown(false).default({}), // default api.services to an empty object
     defaultParameters: Joi.object().keys({

service/configurations/Interpolation.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class Language extends ServiceConfiguration {
+  constructor(o) {
+    super('interpolation', o);
+  }
+
+  getParameters(req, hit) {
+    return {
+      number: req.clean.parsed_text.number,
+      street: hit.address_parts.street || req.clean.parsed_text.street,
+      lat: hit.center_point.lat,
+      lon: hit.center_point.lon
+    };
+
+  }
+
+  getUrl(req) {
+    return url.resolve(this.baseUrl, 'search/geojson');
+  }
+
+}
+
+module.exports = Language;

service/configurations/Language.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class Language extends ServiceConfiguration {
+  constructor(o) {
+    super('language', o);
+  }
+
+  getParameters(req, res) {
+    // find all the values for all keys with names that end with '_id'
+    const ids = _.get(res, 'data', []).reduce((acc, doc) => {
+      Array.prototype.push.apply(acc, _.values(_.pickBy(doc.parent, (v, k) => _.endsWith(k, '_id') ) ) );
+      return acc;
+    }, []);
+
+    return {
+      // arrays will be nested, so flatten first, then uniqify, and finally join elements with comma
+      ids: _.uniq(_.flattenDeep(ids)).join(',')
+    };
+
+  }
+
+  getUrl(req) {
+    return url.resolve(this.baseUrl, 'parser/findbyid');
+  }
+
+}
+
+module.exports = Language;

service/interpolation.js

@@ -1,118 +0,0 @@
-
-var logger = require( 'pelias-logger' ).get( 'api' ),
-    request = require( 'superagent' ),
-    peliasConfig = require( 'pelias-config' );
-
-/**
-
-  street address interpolation service client
-
-  this file provides several different 'transports' which can be used to access the interpolation
-  service, either directly from disk or via a network connnection.
-
-  the exported method for this module checks pelias-config for a configuration block such as:
-
-  "interpolation": {
-    "client": {
-      "adapter": "http",
-      "host": "http://localhost:4444"
-    }
-  }
-
-  for more info on running the service see: https://github.com/pelias/interpolation
-
-**/
-
-/**
-  NullTransport
-
-  disables the service completely
-**/
-function NullTransport(){}
-NullTransport.prototype.query = function( coord, number, street, cb ){
-  cb(); // no-op
-};
-
-/**
-  RequireTransport
-
-  allows the api to be used by simply requiring the module
-**/
-function RequireTransport( addressDbPath, streetDbPath ){
-  try {
-    var lib = require('pelias-interpolation'); // lazy load dependency
-    this.query = lib.api.search( addressDbPath, streetDbPath );
-  } catch( e ){
-    logger.error( 'RequireTransport: failed to connect to interpolation service' );
-  }
-}
-RequireTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'interpolation: transport not connected' );
-};
-
-/**
-  HttpTransport
-
-  allows the api to be used via a remote web service
-**/
-function HttpTransport( host, settings ){
-  this.query = function( coord, number, street, cb ){
-    request
-      .get( host + '/search/geojson' )
-      .set( 'Accept', 'application/json' )
-      .query({ lat: coord.lat, lon: coord.lon, number: number, street: street })
-      .timeout( settings && settings.timeout || 1000 )
-      .end( function( err, res ){
-        if( err || !res ){ return cb( err ); }
-        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
-        return cb( null, res.body );
-      });
-  };
-}
-HttpTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'interpolation: transport not connected' );
-};
-
-/**
-  Setup
-
-  allows instantiation of transport depending on configuration and preference
-**/
-module.exports.search = function setup(){
-
-  // user config
-  var config = peliasConfig.generate();
-
-  // ensure config variables set correctly
-  if( !config.hasOwnProperty('interpolation') || !config.interpolation.hasOwnProperty('client') ){
-    logger.warn( 'interpolation: configuration not found' );
-  }
-
-  // valid configuration found
-  else {
-
-    // get adapter settings from config
-    var settings = config.interpolation.client;
-
-    // http adapter
-    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
-      logger.info( 'interpolation: using http transport:', settings.host );
-      if( settings.hasOwnProperty('timeout') ){
-        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
-      }
-      return new HttpTransport( settings.host );
-    }
-
-    // require adapter
-    else if( 'require' === settings.adapter ){
-      if( settings.hasOwnProperty('streetdb') && settings.hasOwnProperty('addressdb') ){
-        logger.info( 'interpolation: using require transport' );
-        return new RequireTransport( settings.addressdb, settings.streetdb );
-      }
-    }
-  }
-
-  // default adapter
-  logger.info( 'interpolation: using null transport' );
-  return new NullTransport();
-};

service/language.js

@@ -1,93 +0,0 @@
-
-var logger = require( 'pelias-logger' ).get( 'api' ),
-    request = require( 'superagent' ),
-    peliasConfig = require( 'pelias-config' );
-
-/**
-
-  language subsitution service client
-
-  this file provides a 'transport' which can be used to access the language
-  service via a network connnection.
-
-  the exported method for this module checks pelias-config for a configuration block such as:
-
-  "language": {
-    "client": {
-      "adapter": "http",
-      "host": "http://localhost:6100"
-    }
-  }
-
-  for more info on running the service see: https://github.com/pelias/placeholder
-
-**/
-
-/**
-  NullTransport
-
-  disables the service completely
-**/
-function NullTransport(){}
-NullTransport.prototype.query = function( ids, cb ){
-  cb(); // no-op
-};
-
-/**
-  HttpTransport
-
-  allows the api to be used via a remote web service
-**/
-function HttpTransport( host, settings ){
-  this.query = function( ids, cb ){
-    request
-      .get( host + '/parser/findbyid' )
-      .set( 'Accept', 'application/json' )
-      .query({ ids: Array.isArray( ids ) ? ids.join(',') : '' })
-      .timeout( settings && settings.timeout || 1000 )
-      .end( function( err, res ){
-        if( err || !res ){ return cb( err ); }
-        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
-        return cb( null, res.body );
-      });
-  };
-}
-HttpTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'language: transport not connected' );
-};
-
-/**
-  Setup
-
-  allows instantiation of transport depending on configuration and preference
-**/
-module.exports.findById = function setup(){
-
-  // user config
-  var config = peliasConfig.generate();
-
-  // ensure config variables set correctly
-  if( !config.hasOwnProperty('language') || !config.language.hasOwnProperty('client') ){
-    logger.warn( 'language: configuration not found' );
-  }
-
-  // valid configuration found
-  else {
-
-    // get adapter settings from config
-    var settings = config.language.client;
-
-    // http adapter
-    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
-      logger.info( 'language: using http transport:', settings.host );
-      if( settings.hasOwnProperty('timeout') ){
-        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
-      }
-      return new HttpTransport( settings.host );
-    }
-  }
-
-  // default adapter
-  logger.info( 'language: using null transport' );
-  return new NullTransport();
-};

test/unit/controller/predicates/has_any_parsed_text_property.js

@@ -1,94 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-const has_any_parsed_text_property = require('../../../../controller/predicates/has_any_parsed_text_property');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = (test, common) => {
-  test('valid interface', (t) => {
-    t.ok(_.isFunction(has_any_parsed_text_property), 'has_any_parsed_text_property is a function');
-    t.end();
-  });
-};
-
-module.exports.tests.true_conditions = (test, common) => {
-  test('defined request.clean.parsed_text.property should return true', (t) => {
-    const req = {
-      clean: {
-        parsed_text: {
-          property: 'value'
-        }
-      }
-    };
-
-    t.ok(has_any_parsed_text_property('property')(req));
-    t.end();
-
-  });
-
-  test('clean.parsed_text with any property should return true ', (t) => {
-    const req = {
-      clean: {
-        parsed_text: {
-          property2: 'value2',
-          property3: 'value3'
-        }
-      }
-    };
-
-    t.ok(has_any_parsed_text_property('property1', 'property3')(req));
-    t.end();
-
-  });
-
-};
-
-module.exports.tests.false_conditions = (test, common) => {
-  test('undefined request should return false', (t) => {
-    t.notOk(has_any_parsed_text_property('property')());
-    t.end();
-
-  });
-
-  test('undefined request.clean should return false', (t) => {
-    const req = {};
-
-    t.notOk(has_any_parsed_text_property('property')(req));
-    t.end();
-
-  });
-
-  test('undefined request.clean.parsed_text should return false', (t) => {
-    const req = {
-      clean: {}
-    };
-
-    t.notOk(has_any_parsed_text_property('property')(req));
-    t.end();
-
-  });
-
-  test('request.clean.parsed_text with none of the supplied properties should return false', (t) => {
-    const req = {
-      clean: {
-        parsed_text: {}
-      }
-    };
-
-    t.notOk(has_any_parsed_text_property('property1', 'property2')(req));
-    t.end();
-
-  });
-
-};
-
-module.exports.all = (tape, common) => {
-  function test(name, testFunction) {
-    return tape(`GET /has_any_parsed_text_property ${name}`, testFunction);
-  }
-
-  for( const testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/controller/predicates/has_parsed_text_properties.js

@@ -0,0 +1,163 @@
+'use strict';
+
+const _ = require('lodash');
+const has_parsed_text_properties = require('../../../../controller/predicates/has_parsed_text_properties');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.ok(_.isFunction(has_parsed_text_properties.all), 'has_parsed_text_properties.all is a function');
+    t.ok(_.isFunction(has_parsed_text_properties.any), 'has_parsed_text_properties.any is a function');
+    t.end();
+  });
+
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('all: defined request.clean.parsed_text.property should return true', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property: 'value'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: clean.parsed_text with any property should return true ', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property1: 'value1',
+          property2: 'value2'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.all('property2', 'property1')(req));
+    t.end();
+
+  });
+
+  test('any: defined request.clean.parsed_text.property should return true', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property: 'value'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: clean.parsed_text with any property should return true ', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property2: 'value2',
+          property3: 'value3'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.any('property1', 'property3')(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('all: undefined request should return false', (t) => {
+    t.notOk(has_parsed_text_properties.all('property')());
+    t.end();
+
+  });
+
+  test('all: undefined request.clean should return false', (t) => {
+    const req = {};
+
+    t.notOk(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: undefined request.clean.parsed_text should return false', (t) => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: request.clean.parsed_text with none of the supplied properties should return false', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property1: 'value1'
+        }
+      }
+    };
+
+    t.notOk(has_parsed_text_properties.all('property1', 'property2')(req));
+    t.end();
+
+  });
+
+  test('any: undefined request should return false', (t) => {
+    t.notOk(has_parsed_text_properties.any('property')());
+    t.end();
+
+  });
+
+  test('any: undefined request.clean should return false', (t) => {
+    const req = {};
+
+    t.notOk(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: undefined request.clean.parsed_text should return false', (t) => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: request.clean.parsed_text with none of the supplied properties should return false', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {}
+      }
+    };
+
+    t.notOk(has_parsed_text_properties.any('property1', 'property2')(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /has_parsed_text_properties ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/middleware/changeLanguage.js

@@ -1,256 +1,235 @@
+'use strict';
 
-var fs = require('fs'),
+const setup = require('../../../middleware/changeLanguage');
-    tmp = require('tmp'),
-    setup = require('../../../middleware/changeLanguage');
 const proxyquire =  require('proxyquire').noCallThru();
-
+const _  = require('lodash');
-// load middleware using the default pelias config
-var load = function(){
-  // adapter is driven by config
-  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-  process.env.PELIAS_CONFIG = tmpfile;
-  var middleware = setup();
-  delete process.env.PELIAS_CONFIG;
-  return middleware;
-};
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.interface = (test, common) => {
-  test('valid interface', function(t) {
+  test('valid interface', t => {
-    var middleware = load();
+    t.equal(typeof setup, 'function', 'setup is a function');
-    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(typeof setup(), 'function', 'setup returns a controller');
-    t.equal(middleware.length, 3, 'middleware is a function');
     t.end();
   });
 };
 
-module.exports.tests.isLanguageChangeRequired = function(test, common) {
+module.exports.tests.early_exit_conditions = (test, common) => {
-  test('invalid query - null req/res', function(t) {
+  test('should_execute returning false should not call service', t => {
-    var middleware = load();
+    t.plan(2, 'should_execute will assert 2 things');
-    middleware(null, null, t.end);
-  });
-
-  test('invalid query - no results', function(t) {
-    var req = { language: { iso6393: 'spa' } };
-    var res = {};
 
-    var middleware = load();
+    const service = () => {
-    middleware(req, res, function(){
+      t.fail('service should not have been called');
-      t.deepEqual( req, { language: { iso6393: 'spa' } } );
+    };
-      t.deepEqual( res, {} );
-      t.end();
-    });
-  });
 
-  test('invalid query - empty results', function(t) {
+    const should_execute = (req, res) => {
-    var req = { language: { iso6393: 'spa' } };
+      t.deepEquals(req, { a: 1 });
-    var res = { data: [] };
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
 
-    var middleware = load();
+    const controller = setup(service, should_execute);
-    middleware(req, res, function(){
-      t.deepEqual( req, { language: { iso6393: 'spa' } } );
-      t.deepEqual( res, { data: [] } );
-      t.end();
-    });
-  });
 
-  test('invalid query - no target language', function(t) {
+    controller({ a: 1 }, { b: 2 }, () => { });
-    var req = {};
-    var res = { data: [] };
 
-    var middleware = load();
-    middleware(req, res, function(){
-      t.deepEqual( req, {} );
-      t.deepEqual( res, { data: [] } );
-      t.end();
-    });
   });
+
 };
 
-// check the service is called and response mapped correctly
+module.exports.tests.error_conditions = (test, common) => {
-module.exports.tests.miss = function(test, common) {
+  test('service error should log and call next', t => {
-  test('miss', function(t) {
+    // (2) req/res were passed to service
+    // (1) error was logged
+    // (1) res was not modified
+    t.plan(4);
+
+    const service = (req, res, callback) => {
+      t.deepEquals(req, { a: 1 } );
+      t.deepEquals(res, { b: 2 } );
+      callback('this is an error');
+    };
 
-    var req = { language: { iso6393: 'spa' } };
+    const logger = require('pelias-mock-logger')();
-    var res = { data: [
-      {
-        layer: 'locality',
-        name: { default: 'London' },
-        parent: {
-          locality_id: [ 101750367 ],
-          locality: [ 'London' ]
-        }
-      },
-      {
-        layer: 'example',
-        name: { default: 'London' },
-        parent: {
-          locality_id: [ 101735809 ],
-          locaity: [ 'London' ]
-        }
-      }
-    ]};
 
-    var middleware = load();
+    const controller = proxyquire('../../../middleware/changeLanguage', {
+      'pelias-logger': logger
+    })(service, () => true);
 
-    // mock out the transport
+    const req = { a: 1 };
-    middleware.transport.query = function mock( ids, cb ){
+    const res = { b: 2 };
-      t.deepEqual( ids, [ '101735809', '101750367' ] );
-      t.equal( typeof cb, 'function' );
-      cb( 'error' );
-    };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.ok(logger.isErrorMessage('this is an error'));
-        {
+      t.deepEquals(res, { b: 2 }, 'res should not have been modified');
-          layer: 'locality',
-          name: { default: 'London' },
-          parent: {
-            locality_id: [ 101750367 ],
-            locality: [ 'London' ]
-          }
-        },
-        {
-          layer: 'example',
-          name: { default: 'London' },
-          parent: {
-            locality_id: [ 101735809 ],
-            locaity: [ 'London' ]
-          }
-        }
-      ]});
-      t.end();
     });
+
   });
-};
 
-// check the service is called and response mapped correctly
+};
-module.exports.tests.hit = function(test, common) {
-  test('hit', function(t) {
 
-    var req = { language: { iso6393: 'spa' } };
+module.exports.tests.success_conditions = (test, common) => {
-    var res = { data: [
+  test('translations should be mapped in', t => {
-      {
+    // (2) req/res were passed to service
-        layer: 'locality',
+    // (1) error was logged
-        name: { default: 'London' },
+    // (1) res was not modified
-        parent: {
+    // t.plan(4);
-          locality_id: [ 101750367 ],
+
-          locality: [ 'London' ]
+    const service = (req, res, callback) => {
+      const response = {
+        '1': {
+          names: {
+            'requested language': [
+              'replacement name for layer1'
+            ],
+            // this should be ignored
+            'another language': [
+              'name in another language'
+            ]
           }
         },
-      {
+        '2': {
-        layer: 'example',
+          names: {
-        name: { default: 'London' },
+            'requested language': [
-        parent: {
+              'replacement name for layer2',
-          locality_id: [ 101735809 ],
+              // this should be ignored
-          locality: [ 'London' ]
+              'another replacement name for layer2'
+            ]
+          }
+        },
+        '3': {
+          names: {
+            'requested language': [
+              'replacement name 1 for layer3'
+            ]
           }
+        },
+        '4': {
+          names: {
+            'requested language': [
+              'replacement name 2 for layer3'
+            ]
           }
-    ]};
+        },
-
+        '10': {
-    var middleware = load();
+          // has names but not in the requested language
-
+          names: {
-    // mock out the transport
+            'another language': [
-    middleware.transport.query = function mock( ids, cb ){
+              'replacement name for layer4'
-      t.deepEqual( ids, [ '101735809', '101750367' ] );
+            ]
-      t.equal( typeof cb, 'function' );
-      cb( null, {
-        '101750367': {
-          'names': {
-            'default':['London'],
-            'chi':['倫敦'],
-            'spa':['Londres'],
-            'eng':['London'],
-            'hin':['लंदन'],
-            'ara':['لندن'],
-            'por':['Londres'],
-            'ben':['লন্ডন'],
-            'rus':['Лондон'],
-            'jpn':['ロンドン'],
-            'kor':['런던']
           }
         },
-        '101735809': {
+        '11': {
-          'names':{
+          // no names
-            'default':['London'],
+        }
-            'eng':['London']
+      };
+
+      callback(null, response);
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/changeLanguage', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        lang: {
+          iso6393: 'requested language'
         }
       }
-      });
     };
 
-    middleware(req, res, function(){
+    const res = {
-      t.deepEqual( res, { data: [
+      data: [
+        // doc with 2 layer names that will be changed
         {
-          layer: 'locality',
+          name: {
-          name: { default: 'Londres' },
+            default: 'original name for 1st result'
+          },
+          layer: 'layer1',
           parent: {
-            locality_id: [ 101750367 ],
+            layer1_id: ['1'],
-            locality: [ 'Londres' ]
+            layer1: ['original name for layer1'],
+            layer2_id: ['2'],
+            layer2: ['original name for layer2']
           }
         },
+        // not sure how this would sneak in but check anyway
+        undefined,
+        // doc w/o parent
+        {},
+        // doc with only 1 layer name that will be changed and no default name change
         {
-          layer: 'example',
+          name: {
-          name: { default: 'London' },
+            default: 'original name for 2nd result'
+          },
+          layer: 'layer10',
           parent: {
-            locality_id: [ 101735809 ],
+            layer3_id: ['3', '4'],
-            locality: [ 'London' ]
+            layer3: ['original name 1 for layer3', 'original name 2 for layer3'],
+            // requested language not found for this id
+            layer10_id: ['10'],
+            layer10: ['original name for layer10'],
+            // no names for this id
+            layer11_id: ['11'],
+            layer11: ['original name for layer11'],
+            // no translations for this id
+            layer12_id: ['12'],
+            layer12: ['original name for layer12'],
+            // undefined id, will be skipped
+            layer13_id: [undefined],
+            layer13: ['original name for layer13']
           }
         }
-      ]});
+      ]
-      t.end();
+    };
-    });
-  });
 
-  test('empty array name translation should not change the value', t => {
+    controller(req, res, () => {
-    t.plan(2);
+      t.ok(logger.isDebugMessage('[language] [debug] missing translation requested language 10'));
+      t.ok(logger.isDebugMessage('[language] [debug] missing translation requested language 11'));
+      t.ok(logger.isDebugMessage('[language] [debug] failed to find translations for 12'));
 
-    const req = { language: { iso6393: 'ISO3 value' } };
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
-    const res = {
+
+      t.deepEquals(res, {
         data: [
           {
-          layer: 'locality',
+            name: {
-          name: { default: 'original name' },
+              default: 'replacement name for layer1'
+            },
+            layer: 'layer1',
             parent: {
-            locality_id: [ 123 ],
+              layer1_id: ['1'],
-            locality: [ 'original name' ]
+              layer1: ['replacement name for layer1'],
-          }
+              layer2_id: ['2'],
-        }
+              layer2: ['replacement name for layer2']
-      ]
-    };
-
-    const changeLanguage = proxyquire('../../../middleware/changeLanguage', {
-      '../service/language': {
-        findById: () => ({
-          query: (ids, callback) => {
-            t.deepEquals(ids, ['123']);
-            callback(null, {
-              '123': {
-                'names': {
-                  'ISO3 value':[]
-                }
-              }
-            });
-          }
-        })
             }
-    })();
+          },
-
+          undefined,
-    changeLanguage(req, res, () => {
+          {},
-      t.deepEqual( res, { data: [
           {
-          layer: 'locality',
             name: {
-            default: 'original name'
+              default: 'original name for 2nd result'
             },
+            layer: 'layer10',
             parent: {
-            locality_id: [ 123 ],
+              layer3_id: ['3', '4'],
-            locality: [ 'original name' ]
+              layer3: ['replacement name 1 for layer3', 'replacement name 2 for layer3'],
+              layer10_id: ['10'],
+              layer10: ['original name for layer10'],
+              layer11_id: ['11'],
+              layer11: ['original name for layer11'],
+              layer12_id: ['12'],
+              layer12: ['original name for layer12'],
+              layer13_id: [undefined],
+              layer13: ['original name for layer13']
             }
           }
-      ]});
+        ]
+      });
+
+      t.end();
 
     });
 
@@ -258,13 +237,13 @@ module.exports.tests.hit = function(test, common) {
 
 };
 
-module.exports.all = function (tape, common) {
+module.exports.all = (tape, common) => {
 
   function test(name, testFunction) {
-    return tape('[middleware] changeLanguage: ' + name, testFunction);
+    return tape(`GET /changeLanguage ${name}`, testFunction);
   }
 
-  for( var testCase in module.exports.tests ){
+  for( const testCase in module.exports.tests ){
     module.exports.tests[testCase](test, common);
   }
 };

test/unit/middleware/interpolate.js

@@ -1,275 +1,687 @@
+'use strict';
 
-var fs = require('fs'),
+const setup = require('../../../middleware/interpolate');
-    tmp = require('tmp'),
+const proxyquire =  require('proxyquire').noCallThru();
-    setup = require('../../../middleware/interpolate');
+const _  = require('lodash');
-
-// load middleware using the default pelias config
-var load = function(){
-  // adapter is driven by config
-  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-  process.env.PELIAS_CONFIG = tmpfile;
-  var middleware = setup();
-  delete process.env.PELIAS_CONFIG;
-  return middleware;
-};
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.interface = (test, common) => {
-  test('valid interface', function(t) {
+  test('valid interface', t => {
-    var middleware = load();
+    t.equal(typeof setup, 'function', 'setup is a function');
-    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(typeof setup(), 'function', 'setup returns a controller');
-    t.equal(middleware.length, 3, 'middleware is a function');
     t.end();
   });
 };
 
-module.exports.tests.isAddressQuery = function(test, common) {
+module.exports.tests.early_exit_conditions = (test, common) => {
-  test('invalid address query - no parsed text', function(t) {
+  test('should_execute returning false should not call service', t => {
-    var req = { clean: {} };
+    t.plan(3, 'should_execute will assert 2 things + 1 for next() was called');
+
+    const service = () => {
+      t.fail('service should not have been called');
+    };
+
+    const should_execute = (req, res) => {
+      t.deepEquals(req, { a: 1 });
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
+
+    const controller = setup(service, should_execute);
+
+    controller({ a: 1 }, { b: 2 }, () => {
+      t.pass('next was called');
+    });
 
-    var middleware = load();
-    middleware(req, null, t.end);
   });
 
-  test('invalid address query - no number', function(t) {
+};
-    var req = { clean: {
+
-      parsed_text: {
+module.exports.tests.error_conditions = (test, common) => {
-        street: 'sesame st'
+  test('service error string should log and not modify any results', t => {
-      }}
+    t.plan(2);
+
+    const service = (req, res, callback) => {
+      callback('this is an error', {
+        properties: {
+          number: 17,
+          source: 'OSM',
+          source_id: 'openstreetmap source id',
+          lat: 12.121212,
+          lon: 21.212121
+        }
+      });
+
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = { a: 1 };
+    const res = {
+      data: [
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {},
+          // bounding_box should be removed
+          bounding_box: {}
+        }
+      ]
     };
 
-    var middleware = load();
+    controller(req, res, () => {
-    middleware(req, null, t.end);
+      t.ok(logger.isErrorMessage('[middleware:interpolation] this is an error'));
+
+      t.deepEquals(res, {
+        data: [
+          {
+            id: 1,
+            layer: 'street',
+            name: {
+              default: 'street name 1'
+            },
+            address_parts: {},
+            // bounding_box should be removed
+            bounding_box: {}
+          }
+        ]
+      }, 'res should not have been modified');
+
+    });
+
+  });
+
+  test('service error object should log message and not modify any results', t => {
+    t.plan(2);
+
+    const service = (req, res, callback) => {
+      callback({ message: 'this is an error' }, {
+        properties: {
+          number: 17,
+          source: 'OSM',
+          source_id: 'openstreetmap source id',
+          lat: 12.121212,
+          lon: 21.212121
+        }
       });
 
-  test('invalid address query - no street', function(t) {
-    var req = { clean: {
-      parsed_text: {
-        number: '1',
-      }}
     };
 
-    var middleware = load();
+    const logger = require('pelias-mock-logger')();
-    middleware(req, null, t.end);
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = { a: 1 };
+    const res = {
+      data: [
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {},
+          // bounding_box should be removed
+          bounding_box: {}
+        }
+      ]
+    };
+
+    controller(req, res, () => {
+      t.ok(logger.isErrorMessage('[middleware:interpolation] this is an error'));
+
+      t.deepEquals(res, {
+        data: [
+          {
+            id: 1,
+            layer: 'street',
+            name: {
+              default: 'street name 1'
+            },
+            address_parts: {},
+            // bounding_box should be removed
+            bounding_box: {}
+          }
+        ]
+      }, 'res should not have been modified');
+
+    });
+
   });
+
 };
 
-// test results are correctly mapped to the transport
+module.exports.tests.success_conditions = (test, common) => {
-module.exports.tests.map = function(test, common) {
+  test('undefined res should not cause errors', t => {
-  test('documents mapped to transport: no hits', function(t) {
+    const service = (req, res, callback) => {
-    var req = { clean: {
+      t.fail('should not have been called');
-      parsed_text: {
+    };
-        number: '1',
+
-        street: 'sesame st'
+    const logger = require('pelias-mock-logger')();
-      }}
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
     };
-    var res = { data: [] };
 
-    var middleware = load();
+    controller(req, undefined, () => {
-    middleware(req, res, function(){
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
-      t.deepEqual( res, { data: [] } );
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
       t.end();
+
     });
+
   });
-  test('documents mapped to transport: no street layer hits', function(t) {
+
-    var req = { clean: {
+  test('undefined res.data should not cause errors', t => {
-      parsed_text: {
+    const service = (req, res, callback) => {
-        number: '1',
+      t.fail('should not have been called');
-        street: 'sesame st'
-      }}
     };
-    var res = { data: [{ layer: 'foo' }] };
 
-    var middleware = load();
+    const logger = require('pelias-mock-logger')();
-    middleware(req, res, function(){
+
-      t.deepEqual( res, { data: [{ layer: 'foo' }] } );
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
+    };
+
+    const res = {};
+
+    controller(req, res, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      t.deepEquals(res, {});
+
       t.end();
+
     });
+
   });
-};
 
-// check the service is called and response mapped correctly
+  test('interpolated results should be mapped in', t => {
-module.exports.tests.miss = function(test, common) {
+    const service = (req, res, callback) => {
-  test('miss', function(t) {
+      if (res.id === 1) {
+        callback(null, {
+          properties: {
+            number: 17,
+            source: 'Source Abbr 1',
+            source_id: 'source 1 source id',
+            lat: 12.121212,
+            lon: 21.212121
+          }
+        });
+
+      } else if (res.id === 3) {
+        callback(null, {
+          properties: {
+            number: 18,
+            source: 'Source Abbr 2',
+            source_id: 'source 2 source id',
+            lat: 13.131313,
+            lon: 31.313131
+          }
+        });
+
+      } else if (res.id === 4) {
+        callback(null, {
+          properties: {
+            number: 19,
+            source: 'non-OSM/OA',
+            source_id: 'mixed source id',
+            lat: 14.141414,
+            lon: 41.414141
+          }
+        });
+
+      } else {
+        t.fail(`unexpected id ${res.id}`);
+
+      }
+
+    };
+
+    const logger = require('pelias-mock-logger')();
 
-    var req = { clean: {
+    const controller = proxyquire('../../../middleware/interpolate', {
-      parsed_text: {
+      'pelias-logger': logger,
-        number: '1',
+      '../helper/type_mapping': {
-        street: 'sesame st'
+        source_mapping: {
-      }}
+          'source abbr 1': ['full source name 1'],
+          'source abbr 2': ['full source name 2']
+        }
+      }
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
     };
-    var res = { data: [
+
+    const res = {
+      data: [
         {
+          id: 1,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 1'
-        name: { default: 'example' }
+          },
+          address_parts: {},
+          // will be replaced
+          source_id: 'original source_id',
+          // bounding_box should be removed
+          bounding_box: {}
+        },
+        {
+          id: 2,
+          layer: 'not street',
+          name: {
+            default: 'name 2'
+          },
+          address_parts: {}
+        },
+        {
+          id: 3,
+          layer: 'street',
+          name: {
+            default: 'street name 3'
+          },
+          address_parts: {}
+        },
+        {
+          id: 4,
+          layer: 'street',
+          name: {
+            default: 'street name 4'
+          },
+          address_parts: {}
         }
-    ]};
+      ]
+    };
 
-    var middleware = load();
+    controller(req, res, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/), 'timing should be info-logged');
 
-    // mock out the transport
+      // test debug messages very vaguely to avoid brittle tests
-    middleware.transport.query = function mock( coord, number, street, cb ){
+      t.ok(logger.isDebugMessage(/^\[interpolation\] \[hit\] this is req.clean.parsed_text \{.+?\}$/),
-      t.deepEqual( coord, res.data[0].center_point );
+        'hits should be debug-logged');
-      t.deepEqual( number, req.clean.parsed_text.number );
-      t.deepEqual( street, res.data[0].address_parts.street );
-      t.equal( typeof cb, 'function' );
-      cb( 'error' );
-    };
 
-    middleware(req, res, function(){
+      t.deepEquals(res, {
-      t.deepEqual( res, { data: [
+        data: [
           {
-          layer: 'street',
+            id: 1,
-          center_point: { lat: 1, lon: 1 },
+            layer: 'address',
-          address_parts: { street: 'sesame rd' },
+            match_type: 'interpolated',
-          name: { default: 'example' }
+            name: {
+              default: '17 street name 1'
+            },
+            source: 'full source name 1',
+            source_id: 'source 1 source id',
+            address_parts: {
+              number: 17
+            },
+            center_point: {
+              lat: 12.121212,
+              lon: 21.212121
             }
-      ]});
+          },
+          {
+            id: 3,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '18 street name 3'
+            },
+            source: 'full source name 2',
+            source_id: 'source 2 source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
+          },
+          {
+            id: 4,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '19 street name 4'
+            },
+            source: 'mixed',
+            source_id: 'mixed source id',
+            address_parts: {
+              number: 19
+            },
+            center_point: {
+              lat: 14.141414,
+              lon: 41.414141
+            }
+          },
+          {
+            id: 2,
+            layer: 'not street',
+            name: {
+              default: 'name 2'
+            },
+            address_parts: {}
+          }
+        ]
+      }, 'hits should be mapped in and res.data sorted with addresses first and non-addresses last');
+
       t.end();
+
     });
+
   });
-};
 
-// check the service is called and response mapped correctly
+  test('interpolation result without source_id should remove all together', t => {
-module.exports.tests.hit = function(test, common) {
+    const service = (req, res, callback) => {
-  test('hit', function(t) {
+      if (res.id === 1) {
+        callback(null, {
+          properties: {
+            number: 17,
+            source: 'OA',
+            lat: 12.121212,
+            lon: 21.212121
+          }
+        });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
+      }
+
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
 
-    var req = { clean: {
+    const req = {
-      parsed_text: {
+      clean: {
-        number: '1',
+        parsed_text: 'this is req.clean.parsed_text'
-        street: 'sesame st'
+      }
-      }}
     };
-    var res = { data: [
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
         {
+          id: 1,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 1'
-        name: { default: 'street name' },
+          },
-        source_id: '123456'
+          // will be removed
-      }
+          source_id: 'original source_id',
-    ]};
+          address_parts: {}
-
-    var middleware = load();
-
-    // mock out the transport
-    middleware.transport.query = function mock( coord, number, street, cb ){
-      t.deepEqual( coord, res.data[0].center_point );
-      t.deepEqual( number, req.clean.parsed_text.number );
-      t.deepEqual( street, res.data[0].address_parts.street );
-      t.equal( typeof cb, 'function' );
-      cb( null, {
-        properties: {
-          number: '100A',
-          source: 'OSM',
-          source_id: 'way:111111',
-          lat: 22.2,
-          lon: -33.3,
         }
-      });
+      ]
     };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      t.deepEquals(res, {
+        data: [
           {
+            id: 1,
             layer: 'address',
             match_type: 'interpolated',
-          center_point: { lat: 22.2, lon: -33.3 },
+            name: {
-          address_parts: { street: 'sesame rd', number: '100A' },
+              default: '17 street name 1'
-          name: { default: '100A street name' },
+            },
-          source: 'openstreetmap',
+            source: 'openaddresses',
-          source_id: 'way:111111'
+            address_parts: {
+              number: 17
+            },
+            center_point: {
+              lat: 12.121212,
+              lon: 21.212121
+            }
           }
-      ]});
+        ]
+      }, 'interpolation result did not have source_id so removed from source result');
+
       t.end();
+
     });
+
   });
-};
 
-// check the service is called and response mapped correctly
+  test('undefined results should be skipped and not be fatal', t => {
-module.exports.tests.hit = function(test, common) {
+    const service = (req, res, callback) => {
-  test('hit', function(t) {
+      if (res.id === 1) {
+        callback(null, undefined);
+
+      } else if (res.id === 2) {
+        callback(null, {
+          properties: {
+            number: 18,
+            source: 'OA',
+            source_id: 'openaddresses source id',
+            lat: 13.131313,
+            lon: 31.313131
+          }
+        });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
+      }
 
-    var req = { clean: {
-      parsed_text: {
-        number: '1',
-        street: 'sesame st'
-      }}
     };
-    var res = { data: [
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
+    };
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
         {
+          id: 1,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 1'
-        name: { default: 'street name' },
+          },
-        source_id: '123456'
+          address_parts: {}
         },
         {
+          id: 2,
           layer: 'street',
-        center_point: { lat: 2, lon: 2 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 2'
-        name: { default: 'street name' },
+          },
-        source_id: '654321'
+          address_parts: {}
-      }
+        }
-    ]};
+      ]
-
+    };
-    var middleware = load();
+
-
+    controller(req, res, () => {
-    // mock out the transport
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
-    middleware.transport.query = function mock(coord, number, street, cb) {
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
-      if (coord.lat === 2) {
+
-        t.deepEqual(coord, res.data[1].center_point);
+      // test debug messages very vaguely to avoid brittle tests
-        t.deepEqual(number, req.clean.parsed_text.number);
+      t.ok(logger.isDebugMessage('[interpolation] [miss] this is req.clean.parsed_text'));
-        t.deepEqual(street, res.data[1].address_parts.street);
+
-        t.equal(typeof cb, 'function');
+      t.deepEquals(res, {
-        return cb(null, {
+        data: [
+          {
+            id: 2,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '18 street name 2'
+            },
+            source: 'openaddresses',
+            source_id: 'openaddresses source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
+          },
+          {
+            id: 1,
+            layer: 'street',
+            name: {
+              default: 'street name 1'
+            },
+            address_parts: {}
+          }
+        ]
+      }, 'only hits should have been mapped in');
+
+      t.end();
+
+    });
+
+  });
+
+  test('results missing \'properties\' should be skipped and not be fatal', t => {
+    const service = (req, res, callback) => {
+      if (res.id === 1) {
+        callback(null, {});
+
+      } else if (res.id === 2) {
+        callback(null, {
           properties: {
-            number: '100A',
+            number: 18,
-            source: 'OSM',
+            source: 'OA',
-            source_id: 'way:111111',
+            source_id: 'openaddresses source id',
-            lat: 22.2,
+            lat: 13.131313,
-            lon: -33.3,
+            lon: 31.313131
           }
         });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
       }
-      else {
+
-        return cb('miss');
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
+    };
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {}
+        },
+        {
+          id: 2,
+          layer: 'street',
+          name: {
+            default: 'street name 2'
+          },
+          address_parts: {}
         }
+      ]
     };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      // test debug messages very vaguely to avoid brittle tests
+      t.ok(logger.isDebugMessage('[interpolation] [miss] this is req.clean.parsed_text'));
+
+      t.deepEquals(res, {
+        data: [
           {
+            id: 2,
             layer: 'address',
             match_type: 'interpolated',
-          center_point: { lat: 22.2, lon: -33.3 },
+            name: {
-          address_parts: { street: 'sesame rd', number: '100A' },
+              default: '18 street name 2'
-          name: { default: '100A street name' },
+            },
-          source: 'openstreetmap',
+            source: 'openaddresses',
-          source_id: 'way:111111'
+            source_id: 'openaddresses source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
           },
           {
+            id: 1,
             layer: 'street',
-          center_point: { lat: 1, lon: 1 },
+            name: {
-          address_parts: { street: 'sesame rd' },
+              default: 'street name 1'
-          name: { default: 'street name' },
+            },
-          source_id: '123456'
+            address_parts: {}
           }
-      ]});
+        ]
-      t.end();
       });
+
+      t.end();
+
+    }, 'only hits should have been mapped in');
+
   });
-};
 
+};
 
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('[middleware] interpolate: ' + name, testFunction);
+    return tape(`[middleware] interpolate: ${name}`, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/run.js

@@ -18,7 +18,8 @@ var tests = [
   require('./controller/placeholder'),
   require('./controller/search'),
   require('./controller/search_with_ids'),
-  require('./controller/predicates/has_any_parsed_text_property'),
+  require('./controller/predicates/has_parsed_text_properties'),
+  require('./controller/predicates/has_request_parameter'),
   require('./controller/predicates/has_response_data'),
   require('./controller/predicates/has_results_at_layers'),
   require('./controller/predicates/has_request_parameter'),
@@ -95,12 +96,12 @@ var tests = [
   require('./sanitizer/search'),
   require('./sanitizer/defer_to_addressit'),
   require('./sanitizer/wrap'),
+  require('./service/configurations/Interpolation'),
+  require('./service/configurations/Language'),
   require('./service/configurations/PlaceHolder'),
   require('./service/configurations/PointInPolygon'),
   require('./service/mget'),
-  require('./service/search'),
+  require('./service/search')
-  require('./service/interpolation'),
-  require('./service/language')
 ];
 
 tests.map(function(t) {

test/unit/schema.js

@@ -2,6 +2,7 @@
 
 const Joi = require('joi');
 const schema = require('../../schema');
+const _ = require('lodash');
 
 module.exports.tests = {};
 
@@ -20,10 +21,13 @@ module.exports.tests.completely_valid = (test, common) => {
         requestRetries: 19,
         services: {
           pip: {
-            url: 'http://locahost'
+            url: 'http://localhost'
           },
           placeholder: {
-            url: 'http://locahost'
+            url: 'http://localhost'
+          },
+          interpolation: {
+            url: 'http://localhost'
           }
         },
         defaultParameters: {
@@ -537,72 +541,30 @@ module.exports.tests.api_services_validation = (test, common) => {
 
 };
 
-module.exports.tests.placeholder_service_validation = (test, common) => {
+module.exports.tests.service_validation = (test, common) => {
+  // these tests apply for all the individual service definitions
+  const services = ['pip', 'placeholder', 'interpolation'];
+
   test('timeout and retries not specified should default to 250 and 3', (t) => {
+    services.forEach(service => {
       const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          placeholder: {
-            url: 'http://localhost'
-          }
-        }
         },
         esclient: {}
       };
 
-    const result = Joi.validate(config, schema);
+      config.api.services[service] = {
-
+        url: 'http://localhost'
-    t.equals(result.value.api.services.placeholder.timeout, 250);
-    t.equals(result.value.api.services.placeholder.retries, 3);
-    t.end();
-
-  });
-
-  test('when api.services.placeholder is defined, url is required', (t) => {
-    var config = {
-      api: {
-        version: 'version value',
-        indexName: 'index name value',
-        host: 'host value',
-        services: {
-          placeholder: {
-          }
-        }
-      },
-      esclient: {}
-    };
-
-    const result = Joi.validate(config, schema);
-
-    t.equals(result.error.details.length, 1);
-    t.equals(result.error.details[0].message, '"url" is required');
-    t.end();
-
-  });
-
-  test('non-string api.services.placeholder.url should throw error', (t) => {
-    [null, 17, {}, [], true].forEach((value) => {
-      var config = {
-        api: {
-          version: 'version value',
-          indexName: 'index name value',
-          host: 'host value',
-          services: {
-            placeholder: {
-              url: value
-            }
-          }
-        },
-        esclient: {}
       };
 
       const result = Joi.validate(config, schema);
 
-      t.equals(result.error.details.length, 1);
+      t.equals(result.value.api.services[service].timeout, 250);
-      t.equals(result.error.details[0].message, '"url" must be a string');
+      t.equals(result.value.api.services[service].retries, 3);
 
     });
 
@@ -610,119 +572,45 @@ module.exports.tests.placeholder_service_validation = (test, common) => {
 
   });
 
-  test('non-http/https api.services.placeholder.url should throw error', (t) => {
+  test('when api.services.service is defined, url is required', (t) => {
-    ['ftp', 'git', 'unknown'].forEach((scheme) => {
+    services.forEach(service => {
-      var config = {
+      const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-          services: {
+          services: {}
-            placeholder: {
-              url: `${scheme}://localhost`
-            }
-          }
         },
         esclient: {}
       };
 
-      const result = Joi.validate(config, schema);
+      config.api.services[service] = {};
-
-      t.equals(result.error.details.length, 1);
-      t.equals(result.error.details[0].message, '"url" must be a valid uri with a scheme matching the https\? pattern');
-
-    });
-
-    t.end();
-
-  });
-
-  test('non-url children of api.services.placeholder should be disallowed', (t) => {
-    var config = {
-      api: {
-        version: 'version value',
-        indexName: 'index name value',
-        host: 'host value',
-        services: {
-          placeholder: {
-            url: 'http://localhost',
-            unknown_property: 'value'
-          }
-        }
-      },
-      esclient: {}
-    };
 
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
-    t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
+      t.equals(result.error.details[0].message, '"url" is required');
-    t.end();
-
     });
 
-};
-
-module.exports.tests.pip_service_validation = (test, common) => {
-  test('timeout and retries not specified should default to 250 and 3', (t) => {
-    const config = {
-      api: {
-        version: 'version value',
-        indexName: 'index name value',
-        host: 'host value',
-        services: {
-          pip: {
-            url: 'http://localhost'
-          }
-        }
-      },
-      esclient: {}
-    };
-
-    const result = Joi.validate(config, schema);
-
-    t.equals(result.value.api.services.pip.timeout, 250);
-    t.equals(result.value.api.services.pip.retries, 3);
     t.end();
 
   });
 
-  test('when api.services.pip is defined, url is required', (t) => {
+  test('non-string api.services.pip.url should throw error', (t) => {
-    var config = {
+    services.forEach(service => {
+      [null, 17, {}, [], true].forEach(value => {
+        const config = {
           api: {
             version: 'version value',
             indexName: 'index name value',
             host: 'host value',
-        services: {
+            services: {}
-          pip: {
-          }
-        }
           },
           esclient: {}
         };
 
-    const result = Joi.validate(config, schema);
+        config.api.services[service] = {
-
-    t.equals(result.error.details.length, 1);
-    t.equals(result.error.details[0].message, '"url" is required');
-    t.end();
-
-  });
-
-  test('non-string api.services.pip.url should throw error', (t) => {
-    [null, 17, {}, [], true].forEach((value) => {
-      var config = {
-        api: {
-          version: 'version value',
-          indexName: 'index name value',
-          host: 'host value',
-          services: {
-            pip: {
           url: value
-            }
-          }
-        },
-        esclient: {}
         };
 
         const result = Joi.validate(config, schema);
@@ -732,26 +620,29 @@ module.exports.tests.pip_service_validation = (test, common) => {
 
       });
 
+    });
+
     t.end();
 
   });
 
   test('non-http/https api.services.pip.url should throw error', (t) => {
+    services.forEach(service => {
       ['ftp', 'git', 'unknown'].forEach((scheme) => {
-      var config = {
+        const config = {
           api: {
             version: 'version value',
             indexName: 'index name value',
             host: 'host value',
-          services: {
+            services: {}
-            pip: {
-              url: `${scheme}://localhost`
-            }
-          }
           },
           esclient: {}
         };
 
+        config.api.services[service] = {
+          url: `${scheme}://localhost`
+        };
+
         const result = Joi.validate(config, schema);
 
         t.equals(result.error.details.length, 1);
@@ -759,51 +650,58 @@ module.exports.tests.pip_service_validation = (test, common) => {
 
       });
 
+    });
+
     t.end();
 
   });
 
   test('non-url children of api.services.pip should be disallowed', (t) => {
-    var config = {
+    services.forEach(service => {
+      const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          pip: {
-            url: 'http://localhost',
-            unknown_property: 'value'
-          }
-        }
         },
         esclient: {}
       };
 
+      config.api.services[service] = {
+        url: 'http://localhost',
+        unknown_property: 'value'
+      };
+
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
       t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
+
+    });
+
     t.end();
 
   });
 
   test('non-number timeout should throw error', (t) => {
+    services.forEach(service => {
       [null, 'string', {}, [], false].forEach((value) => {
         const config = {
           api: {
             version: 'version value',
             indexName: 'index name value',
             host: 'host value',
-          services: {
+            services: {}
-            pip: {
-              url: 'http://localhost',
-              timeout: value
-            }
-          }
           },
           esclient: {}
         };
 
+        config.api.services[service] = {
+          url: 'http://localhost',
+          timeout: value
+        };
+
         const result = Joi.validate(config, schema);
 
         t.equals(result.error.details.length, 1);
@@ -811,75 +709,86 @@ module.exports.tests.pip_service_validation = (test, common) => {
 
       });
 
+    });
+
     t.end();
 
   });
 
   test('non-integer timeout should throw error', (t) => {
+    services.forEach(service => {
       const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          pip: {
-            url: 'http://localhost',
-            timeout: 17.3
-          }
-        }
         },
         esclient: {}
       };
 
+      config.api.services[service] = {
+        url: 'http://localhost',
+        timeout: 17.3
+      };
+
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
       t.equals(result.error.details[0].message, '"timeout" must be an integer');
+
+    });
+
     t.end();
 
   });
 
   test('negative timeout should throw error', (t) => {
+    services.forEach(service => {
       const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          pip: {
-            url: 'http://localhost',
-            timeout: -1
-          }
-        }
         },
         esclient: {}
       };
 
+      config.api.services[service] = {
+        url: 'http://localhost',
+        timeout: -1
+      };
+
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
       t.equals(result.error.details[0].message, '"timeout" must be larger than or equal to 0');
+
+    });
+
     t.end();
 
   });
 
   test('non-number retries should throw error', (t) => {
+    services.forEach(service => {
       [null, 'string', {}, [], false].forEach((value) => {
         const config = {
           api: {
             version: 'version value',
             indexName: 'index name value',
             host: 'host value',
-          services: {
+            services: {}
-            pip: {
-              url: 'http://localhost',
-              retries: value
-            }
-          }
           },
           esclient: {}
         };
 
+        config.api.services[service] = {
+          url: 'http://localhost',
+          retries: value
+        };
+
         const result = Joi.validate(config, schema);
 
         t.equals(result.error.details.length, 1);
@@ -887,54 +796,64 @@ module.exports.tests.pip_service_validation = (test, common) => {
 
       });
 
+    });
+
     t.end();
 
   });
 
   test('non-integer retries should throw error', (t) => {
+    services.forEach(service => {
       const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          pip: {
-            url: 'http://localhost',
-            retries: 17.3
-          }
-        }
         },
         esclient: {}
       };
 
+      config.api.services[service] = {
+        url: 'http://localhost',
+        retries: 17.3
+      };
+
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
       t.equals(result.error.details[0].message, '"retries" must be an integer');
+
+    });
+
     t.end();
 
   });
 
   test('negative retries should throw error', (t) => {
+    services.forEach(service => {
       const config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-        services: {
+          services: {}
-          pip: {
-            url: 'http://localhost',
-            retries: -1
-          }
-        }
         },
         esclient: {}
       };
 
+      config.api.services[service] = {
+        url: 'http://localhost',
+        retries: -1
+      };
+
       const result = Joi.validate(config, schema);
 
       t.equals(result.error.details.length, 1);
       t.equals(result.error.details[0].message, '"retries" must be larger than or equal to 0');
+
+    });
+
     t.end();
 
   });

test/unit/service/configurations/Interpolation.js

@@ -0,0 +1,140 @@
+module.exports.tests = {};
+
+const Interpolation = require('../../../../service/configurations/Interpolation');
+
+module.exports.tests.all = (test, common) => {
+  test('getName should return \'interpolation\'', t => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    t.equals(interpolation.getName(), 'interpolation');
+    t.equals(interpolation.getBaseUrl(), 'http://localhost:1234/');
+    t.equals(interpolation.getTimeout(), 17);
+    t.equals(interpolation.getRetries(), 19);
+    t.end();
+
+  });
+
+  test('getUrl should return value passed to constructor', t => {
+    const configBlob = {
+      url: 'http://localhost:1234'
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    t.equals(interpolation.getUrl(), 'http://localhost:1234/search/geojson');
+    t.end();
+
+  });
+
+  test('getHeaders should return empty object', t => {
+    const configBlob = {
+      url: 'base url'
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    t.deepEquals(interpolation.getHeaders(), {});
+    t.end();
+
+  });
+
+  test('getParameters should return hit.address_parts.street over req.clean.parsed_text.street', t => {
+    const configBlob = {
+      url: 'base url'
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    const req = {
+      clean: {
+        parsed_text: {
+          number: 'parsed number value',
+          street: 'parsed street value'
+        }
+      }
+    };
+
+    const hit = {
+      address_parts: {
+        street: 'hit street value'
+      },
+      center_point: {
+        lat: 12.121212,
+        lon: 21.212121
+      }
+    };
+
+    t.deepEquals(interpolation.getParameters(req, hit), {
+      number: 'parsed number value',
+      street: 'hit street value',
+      lat: 12.121212,
+      lon: 21.212121
+    });
+    t.end();
+
+  });
+
+  test('getParameters should return req.clean.parsed_text.street when hit.address_parts.street unavailable', t => {
+    const configBlob = {
+      url: 'base url'
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    const req = {
+      clean: {
+        parsed_text: {
+          number: 'parsed number value',
+          street: 'parsed street value'
+        }
+      }
+    };
+
+    const hit = {
+      address_parts: {
+      },
+      center_point: {
+        lat: 12.121212,
+        lon: 21.212121
+      }
+    };
+
+    t.deepEquals(interpolation.getParameters(req, hit), {
+      number: 'parsed number value',
+      street: 'parsed street value',
+      lat: 12.121212,
+      lon: 21.212121
+    });
+    t.end();
+
+  });
+
+  test('baseUrl ending in / should not have double /\'s return by getUrl', t => {
+    const configBlob = {
+      url: 'http://localhost:1234/'
+    };
+
+    const interpolation = new Interpolation(configBlob);
+
+    t.deepEquals(interpolation.getUrl(), 'http://localhost:1234/search/geojson');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SERVICE CONFIGURATION /Interpolation ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/service/configurations/Language.js

@@ -0,0 +1,158 @@
+module.exports.tests = {};
+
+const Language = require('../../../../service/configurations/Language');
+
+module.exports.tests.all = (test, common) => {
+  test('getName should return \'language\'', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    t.equals(language.getName(), 'language');
+    t.equals(language.getBaseUrl(), 'http://localhost:1234/');
+    t.equals(language.getTimeout(), 17);
+    t.equals(language.getRetries(), 19);
+    t.end();
+
+  });
+
+  test('getUrl should return value passed to constructor', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    t.equals(language.getUrl(), 'http://localhost:1234/parser/findbyid');
+    t.end();
+
+  });
+
+  test('getParameters should return object with all deduped ids extracted from res', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    const res = {
+      data: [
+        {
+          parent: {
+            layer1_name: 'layer1 name',
+            layer1_id: [1],
+            layer2_name: 'layer2 name',
+            layer2_id: [2]
+          }
+        },
+        {
+          // empty parent
+          parent: {}
+        },
+        {
+          // no parent
+        },
+        {
+          parent: {
+            layer3_name: 'layer3 name',
+            layer3_id: [3],
+            layer4_name: 'layer4 name',
+            // doesn't end with '_id', will be ignored
+            layer4id: [4]
+          }
+        },
+        {
+          parent: {
+            // this is a duplicate id
+            layer1_name: 'layer1 name',
+            layer1_id: [1],
+            // two ids, both should be added
+            layer5_name: 'layer5 name',
+            layer5_id: [5, 6]
+          }
+        }
+      ]
+    };
+
+    t.deepEquals(language.getParameters(undefined, res), { ids: '1,2,3,5,6' });
+    t.end();
+
+  });
+
+  test('getParameters should return empty ids array when res is undefined', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    t.deepEquals(language.getParameters(undefined, undefined), { ids: '' });
+    t.end();
+
+  });
+
+  test('getParameters should return empty ids array when res.data is undefined', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const res = { };
+
+    const language = new Language(configBlob);
+
+    t.deepEquals(language.getParameters(undefined, res), { ids: '' });
+    t.end();
+
+  });
+
+  test('getHeaders should return empty object', (t) => {
+    const configBlob = {
+      url: 'base url',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    t.deepEquals(language.getHeaders(), {});
+    t.end();
+
+  });
+
+  test('baseUrl ending in / should not have double /\'s return by getUrl', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234/',
+      timeout: 17,
+      retries: 19
+    };
+
+    const language = new Language(configBlob);
+
+    t.deepEquals(language.getUrl(), 'http://localhost:1234/parser/findbyid');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SERVICE CONFIGURATION /Language ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/service/interpolation.js

@@ -1,128 +0,0 @@
-
-var fs = require('fs'),
-    tmp = require('tmp'),
-    setup = require('../../../service/interpolation').search;
-
-module.exports.tests = {};
-
-module.exports.tests.interface = function(test, common) {
-  test('valid interface', function(t) {
-    t.equal(typeof setup, 'function', 'setup is a function');
-    t.end();
-  });
-};
-
-// adapter factory
-module.exports.tests.factory = function(test, common) {
-
-  test('http adapter', function(t) {
-    var config = { interpolation: { client: {
-      adapter: 'http',
-      host: 'http://example.com'
-    }}};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'HttpTransport', 'HttpTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 4, 'query function signature');
-    t.end();
-  });
-
-  test('require adapter', function(t) {
-    var config = { interpolation: { client: {
-      adapter: 'require',
-      addressdb: '/tmp/address.db',
-      streetdb: '/tmp/street.db'
-    }}};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'RequireTransport', 'RequireTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 4, 'query function signature');
-    t.end();
-  });
-
-  test('null adapter', function(t) {
-    var config = { interpolation: { client: {
-      adapter: 'null'
-    }}};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 4, 'query function signature');
-    t.end();
-  });
-
-  test('default adapter', function(t) {
-    var config = {};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 4, 'query function signature');
-    t.end();
-  });
-
-};
-
-// null transport
-module.exports.tests.NullTransport = function(test, common) {
-
-  test('null transport', function(t) {
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    // test null transport performs a no-op
-    adapter.query( null, null, null, function( err, res ){
-      t.equal(err, undefined, 'no-op');
-      t.equal(res, undefined, 'no-op');
-      t.end();
-    });
-  });
-
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('SERVICE interpolation', testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/service/language.js

@@ -1,107 +0,0 @@
-
-var fs = require('fs'),
-    tmp = require('tmp'),
-    setup = require('../../../service/language').findById;
-
-module.exports.tests = {};
-
-module.exports.tests.interface = function(test, common) {
-  test('valid interface', function(t) {
-    t.equal(typeof setup, 'function', 'setup is a function');
-    t.end();
-  });
-};
-
-// adapter factory
-module.exports.tests.factory = function(test, common) {
-
-  test('http adapter', function(t) {
-    var config = { language: { client: {
-      adapter: 'http',
-      host: 'http://example.com'
-    }}};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'HttpTransport', 'HttpTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 2, 'query function signature');
-    t.end();
-  });
-
-  test('null adapter', function(t) {
-    var config = { language: { client: {
-      adapter: 'null'
-    }}};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 2, 'query function signature');
-    t.end();
-  });
-
-  test('default adapter', function(t) {
-    var config = {};
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
-    t.equal(typeof adapter, 'object', 'adapter is an object');
-    t.equal(typeof adapter.query, 'function', 'query is a function');
-    t.equal(adapter.query.length, 2, 'query function signature');
-    t.end();
-  });
-
-};
-
-// null transport
-module.exports.tests.NullTransport = function(test, common) {
-
-  test('null transport', function(t) {
-
-    // adapter is driven by config
-    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-    fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-    process.env.PELIAS_CONFIG = tmpfile;
-    var adapter = setup();
-    delete process.env.PELIAS_CONFIG;
-
-    // test null transport performs a no-op
-    adapter.query( null, function( err, res ){
-      t.equal(err, undefined, 'no-op');
-      t.equal(res, undefined, 'no-op');
-      t.end();
-    });
-  });
-
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('SERVICE language', testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};