Merge pull request #953 from pelias/master

Merge master into staging

controller/libpostal.js

@@ -0,0 +1,31 @@
+const text_analyzer = require('pelias-text-analyzer');
+const _ = require('lodash');
+const iso3166 = require('iso3166-1');
+
+function setup(should_execute) {
+  function controller( req, res, next ){
+    // bail early if req/res don't pass conditions for execution
+    if (!should_execute(req, res)) {
+      return next();
+    }
+
+    // parse text with query parser
+    const parsed_text = text_analyzer.parse(req.clean.text);
+
+    if (parsed_text !== undefined) {
+      // if a known ISO2 country was parsed, convert it to ISO3
+      if (_.has(parsed_text, 'country') && iso3166.is2(_.toUpper(parsed_text.country))) {
+        parsed_text.country = iso3166.to3(_.toUpper(parsed_text.country));
+      }
+
+      req.clean.parsed_text = parsed_text;
+    }
+
+    return next();
+
+  }
+
+  return controller;
+}
+
+module.exports = setup;

controller/placeholder.js

@@ -59,16 +59,16 @@ function hasName(result) {
 }
 
 // filter that passes only results that match on requested layers
-// passes everything if req.clean.layers is not found
 function getLayersFilter(clean) {
-  if (_.isEmpty(_.get(clean, 'layers', []))) {
+  // passes everything if:
-    return _.constant(true);
+  // - req.clean.layers is empty
+  // - req.clean.parsed_text.street is available
+  if (_.isEmpty(_.get(clean, 'layers', [])) || _.has(clean, ['parsed_text', 'street'])) {
+    return () => true;
   }
 
   // otherwise return a function that checks for set inclusion of a result placetype
-  return (result) => {
+  return (result) => _.includes(clean.layers, result.placetype);
-    return _.includes(clean.layers, result.placetype);
-  };
 
 }
 
@@ -85,20 +85,20 @@ function atLeastOneLineageMatchesBoundaryCountry(boundaryCountry, result) {
 }
 
 // return a function that detects if a result has at least one lineage in boundary.country
-// if there's no boundary.country, return a function that always returns true
+function getBoundaryCountryFilter(clean, do_geometric_filters_apply) {
-function getBoundaryCountryFilter(clean) {
+  if ( do_geometric_filters_apply && _.has(clean, 'boundary.country') ) {
-  if (_.has(clean, 'boundary.country')) {
     return _.partial(atLeastOneLineageMatchesBoundaryCountry, clean['boundary.country']);
   }
 
-  return _.constant(true);
+  // there's no boundary.country filter, so return a function that always returns true
+  return () => true;
 
 }
 
 // return a function that detects if a result is inside a bbox if a bbox is available
-// if there's no bbox, return a function that always returns true
+function getBoundaryRectangleFilter(clean, do_geometric_filters_apply) {
-function getBoundaryRectangleFilter(clean) {
+  // check to see if boundary.rect.min_lat/min_lon/max_lat/max_lon are all available
-  if (['min_lat', 'min_lon', 'max_lat', 'max_lon'].every((f) => {
+  if (do_geometric_filters_apply && ['min_lat', 'min_lon', 'max_lat', 'max_lon'].every((f) => {
     return _.has(clean, `boundary.rect.${f}`);
   })) {
     const polygon = [
@@ -107,20 +107,22 @@ function getBoundaryRectangleFilter(clean) {
       { latitude: clean['boundary.rect.max_lat'], longitude: clean['boundary.rect.max_lon'] },
       { latitude: clean['boundary.rect.min_lat'], longitude: clean['boundary.rect.max_lon'] }
     ];
+    // isPointInside takes polygon last, so create a function that has it pre-populated
     const isPointInsidePolygon = _.partialRight(geolib.isPointInside, polygon);
 
     return _.partial(isInsideGeometry, isPointInsidePolygon);
 
   }
 
-  return _.constant(true);
+  // there's no bbox filter, so return a function that always returns true
+  return () => true;
 
 }
 
 // return a function that detects if a result is inside a circle if a circle is available
-// if there's no circle, return a function that always returns true
+function getBoundaryCircleFilter(clean, do_geometric_filters_apply) {
-function getBoundaryCircleFilter(clean) {
+  // check to see if boundary.circle.lat/lon/radius are all available
-  if (['lat', 'lon', 'radius'].every((f) => {
+  if (do_geometric_filters_apply && ['lat', 'lon', 'radius'].every((f) => {
     return _.has(clean, `boundary.circle.${f}`);
   })) {
     const center = {
@@ -128,13 +130,16 @@ function getBoundaryCircleFilter(clean) {
       longitude: clean['boundary.circle.lon']
     };
     const radiusInMeters = clean['boundary.circle.radius'] * 1000;
+
+    // isPointInCircle takes circle/radius last, so create a function that has them pre-populated
     const isPointInCircle = _.partialRight(geolib.isPointInCircle, center, radiusInMeters);
 
     return _.partial(isInsideGeometry, isPointInCircle);
 
   }
 
-  return _.constant(true);
+  // there's no circle filter, so return a function that always returns true
+  return () => true;
 
 }
 
@@ -143,6 +148,7 @@ function isInsideGeometry(f, result) {
   return hasLatLon(result) ? f(getLatLon(result)) : false;
 }
 
+// returns true if hierarchyElement has both name and id
 function placetypeHasNameAndId(hierarchyElement) {
   return !_.isEmpty(_.trim(hierarchyElement.name)) &&
           !_.isEmpty(_.trim(hierarchyElement.id));
@@ -160,7 +166,7 @@ function synthesizeDocs(boundaryCountry, result) {
     logger.error(`could not parse centroid for id ${result.id}`);
   }
 
-  // lodash conformsTo verifies that an object has a property with a certain format
+  // _.conformsTo verifies that an object property has a certain format
   if (_.conformsTo(result.geom, { 'bbox': is4CommaDelimitedNumbers } )) {
     const parsedBoundingBox = result.geom.bbox.split(',').map(_.toFinite);
     doc.setBoundingBox({
@@ -213,7 +219,7 @@ function buildESDoc(doc) {
   return _.extend(esDoc.data, { _id: esDoc._id, _type: esDoc._type });
 }
 
-function setup(placeholderService, should_execute) {
+function setup(placeholderService, do_geometric_filters_apply, should_execute) {
   function controller( req, res, next ){
     // bail early if req/res don't pass conditions for execution
     if (!should_execute(req, res)) {
@@ -222,15 +228,11 @@ function setup(placeholderService, should_execute) {
 
     placeholderService(req, (err, results) => {
       if (err) {
-        // bubble up an error if one occurred
+        // push err.message or err onto req.errors
-        if (_.isObject(err) && err.message) {
+        req.errors.push( _.get(err, 'message', err));
-          req.errors.push( err.message );
-        } else {
-          req.errors.push( err );
-        }
 
       } else {
-        const boundaryCountry = _.get(req, ['clean', 'boundary.country']);
+        const boundaryCountry = do_geometric_filters_apply ? _.get(req, ['clean', 'boundary.country']) : undefined;
 
         // convert results to ES docs
         // boundary.country filter must happen after synthesis since multiple
@@ -245,13 +247,13 @@ function setup(placeholderService, should_execute) {
                     // filter out results that don't match on requested layer(s)
                     .filter(getLayersFilter(req.clean))
                     // filter out results that don't match on any lineage country
-                    .filter(getBoundaryCountryFilter(req.clean))
+                    .filter(getBoundaryCountryFilter(req.clean, do_geometric_filters_apply))
                     // clean up geom.lat/lon for boundary rect/circle checks
                     .map(numberifyGeomLatLon)
                     // filter out results that aren't in the boundary.rect
-                    .filter(getBoundaryRectangleFilter(req.clean))
+                    .filter(getBoundaryRectangleFilter(req.clean, do_geometric_filters_apply))
                     // filter out results that aren't in the boundary.circle
-                    .filter(getBoundaryCircleFilter(req.clean))
+                    .filter(getBoundaryCircleFilter(req.clean, do_geometric_filters_apply))
                     // convert results to ES docs
                     .map(_.partial(synthesizeDocs, boundaryCountry));
 

controller/predicates/has_parsed_text_properties.js

@@ -0,0 +1,33 @@
+const _ = require('lodash');
+
+// "arguments" is only available in long-form function declarations, cannot be shortened to fat arrow syntax
+// potential improvement:  inject set operator to allow for any/all functionality
+module.exports = {
+  all: function() {
+    // save off property names for future reference
+    const properties = _.values(arguments);
+
+    // return true if ALL of the supplied properties are in clean.parsed_text
+    return request => _.isEmpty(
+      _.difference(
+        _.values(properties),
+        _.keys(_.get(request, ['clean', 'parsed_text'], {}))
+      )
+    );
+
+  },
+  any: function() {
+    // save off property names for future reference
+    const properties = _.values(arguments);
+
+    // return true if ANY of the supplied properties are in clean.parsed_text
+    return request => !_.isEmpty(
+      _.intersection(
+        _.values(properties),
+        _.keys(_.get(request, ['clean', 'parsed_text'], {}))
+      )
+    );
+
+  }
+
+};

controller/predicates/has_request_parameter.js

@@ -0,0 +1,4 @@
+const _ = require('lodash');
+
+// returns true IFF req.clean has a key with the supplied name
+module.exports = (parameter) => (req, res) => (_.has(req, ['clean', parameter]));

controller/predicates/is_addressit_parse.js

@@ -0,0 +1,4 @@
+const _ = require('lodash');
+
+// returns true IFF req.clean.parser is addressit
+module.exports = (req, res) => (_.get(req, 'clean.parser') === 'addressit');

controller/predicates/is_admin_only_analysis.js

@@ -1,13 +1,18 @@
 const _ = require('lodash');
+const Debug = require('../../helper/debug');
+const debugLog = new Debug('controller:predicates:is_admin_only_analysis');
 
 module.exports = (request, response) => {
   if (!request.clean.hasOwnProperty('parsed_text')) {
+    debugLog.push(request, false + '(no parsed_text)');
     return false;
   }
 
   // return true only if all non-admin properties of parsed_text are empty
-  return ['number', 'street', 'query', 'category'].every((prop) => {
+   const is_admin_only_analysis = ['number', 'street', 'query', 'category', 'postalcode'].every((prop) => {
     return _.isEmpty(request.clean.parsed_text[prop]);
   });
 
+  debugLog.push(request, is_admin_only_analysis);
+  return is_admin_only_analysis;
 };

controller/predicates/is_coarse_reverse.js

@@ -1,9 +1,13 @@
 const _ = require('lodash');
-
+const Debug = require('../../helper/debug');
+const debugLog = new Debug('controller:predicates:is_coarse_reverse');
 const non_coarse_layers = ['address', 'street', 'venue'];
 
 module.exports = (req, res) => {
   // returns true if layers is undefined, empty, or contains 'address', 'street', or 'venue'
-  return !_.isEmpty(req.clean.layers) &&
+  const is_coarse_reverse = !_.isEmpty(req.clean.layers) &&
           _.isEmpty(_.intersection(req.clean.layers, non_coarse_layers));
+
+  debugLog.push(req, is_coarse_reverse);
+  return is_coarse_reverse;
 };

controller/predicates/is_only_non_admin_layers.js

@@ -0,0 +1,7 @@
+const _ = require('lodash');
+
+// return true IFF req.clean.layers is empty OR there are non-venue/address/street layers
+module.exports = (req, res) => (
+  !_.isEmpty(_.get(req, 'clean.layers', [])) &&
+  _.isEmpty(_.difference(req.clean.layers, ['venue', 'address', 'street']))
+);

controller/predicates/is_request_sources_only_whosonfirst.js

@@ -0,0 +1,9 @@
+const _ = require('lodash');
+
+// returns true IFF 'whosonfirst' is the only requested source
+module.exports = (req, res) => (
+  _.isEqual(
+    _.get(req, 'clean.sources', []),
+    ['whosonfirst']
+  )
+);

controller/predicates/is_service_enabled.js

@@ -1,5 +0,0 @@
-module.exports = (uri) => {
-  return (request, response) => {
-    return uri !== undefined;
-  };
-};

controller/search.js

@@ -6,6 +6,8 @@ const searchService = require('../service/search');
 const logger = require('pelias-logger').get('api');
 const logging = require( '../helper/logging' );
 const retry = require('retry');
+const Debug = require('../helper/debug');
+const debugLog = new Debug('controller:search');
 
 function isRequestTimeout(err) {
   return _.get(err, 'status') === 408;
@@ -16,7 +18,7 @@ function setup( apiConfig, esclient, query, should_execute ){
     if (!should_execute(req, res)) {
       return next();
     }
-
+    debugLog.beginTimer(req);
     let cleanOutput = _.cloneDeep(req.clean);
     if (logging.isDNT(req)) {
       cleanOutput = logging.removeFields(cleanOutput);
@@ -28,6 +30,7 @@ function setup( apiConfig, esclient, query, should_execute ){
 
     // if there's no query to call ES with, skip the service
     if (_.isUndefined(renderedQuery)) {
+      debugLog.stopTimer(req, 'No query to call ES with. Skipping');
       return next();
     }
 
@@ -60,6 +63,7 @@ function setup( apiConfig, esclient, query, should_execute ){
         // only consider for status 408 (request timeout)
         if (isRequestTimeout(err) && operation.retry(err)) {
           logger.info(`request timed out on attempt ${currentAttempt}, retrying`);
+          debugLog.stopTimer(req, 'request timed out, retrying');
           return;
         }
 
@@ -97,11 +101,16 @@ function setup( apiConfig, esclient, query, should_execute ){
           ];
 
           logger.info(messageParts.join(' '));
+          debugLog.push(req, {queryType: {
+            [renderedQuery.type] : {
+              es_result_count: parseInt(messageParts[2].slice(17, -1))
+            }
+          }});
         }
         logger.debug('[ES response]', docs);
         next();
       });
-
+      debugLog.stopTimer(req);
     });
 
   }

controller/search_with_ids.js

@@ -0,0 +1,113 @@
+const _ = require('lodash');
+
+const searchService = require('../service/search');
+const logger = require('pelias-logger').get('api');
+const logging = require( '../helper/logging' );
+const retry = require('retry');
+
+function isRequestTimeout(err) {
+  return _.get(err, 'status') === 408;
+}
+
+function setup( apiConfig, esclient, query, should_execute ){
+  function controller( req, res, next ){
+    if (!should_execute(req, res)) {
+      return next();
+    }
+
+    const cleanOutput = _.cloneDeep(req.clean);
+    if (logging.isDNT(req)) {
+      logging.removeFields(cleanOutput);
+    }
+    // log clean parameters for stats
+    logger.info('[req]', `endpoint=${req.path}`, cleanOutput);
+
+    const renderedQuery = query(req.clean, res);
+
+    // if there's no query to call ES with, skip the service
+    if (_.isUndefined(renderedQuery)) {
+      return next();
+    }
+
+    // options for retry
+    // maxRetries is from the API config with default of 3
+    // factor of 1 means that each retry attempt will esclient requestTimeout
+    const operationOptions = {
+      retries: _.get(apiConfig, 'requestRetries', 3),
+      factor: 1,
+      minTimeout: _.get(esclient, 'transport.requestTimeout')
+    };
+
+    // setup a new operation
+    const operation = retry.operation(operationOptions);
+
+    // elasticsearch command
+    const cmd = {
+      index: apiConfig.indexName,
+      searchType: 'dfs_query_then_fetch',
+      body: renderedQuery.body
+    };
+
+    logger.debug( '[ES req]', cmd );
+
+    operation.attempt((currentAttempt) => {
+      // query elasticsearch
+      searchService( esclient, cmd, function( err, docs, meta ){
+        // returns true if the operation should be attempted again
+        // (handles bookkeeping of maxRetries)
+        // only consider for status 408 (request timeout)
+        if (isRequestTimeout(err) && operation.retry(err)) {
+          logger.info(`request timed out on attempt ${currentAttempt}, retrying`);
+          return;
+        }
+
+        // if execution has gotten this far then one of three things happened:
+        // - the request didn't time out
+        // - maxRetries has been hit so we're giving up
+        // - another error occurred
+        // in either case, handle the error or results
+
+        // error handler
+        if( err ){
+          // push err.message or err onto req.errors
+          req.errors.push( _.get(err, 'message', err));
+        }
+        else {
+          // log that a retry was successful
+          // most requests succeed on first attempt so this declutters log files
+          if (currentAttempt > 1) {
+            logger.info(`succeeded on retry ${currentAttempt-1}`);
+          }
+
+          // because this is used in response to placeholder, there may already
+          // be results.  if there are no results from this ES call, don't overwrite
+          // what's already there from placeholder.
+          if (!_.isEmpty(docs)) {
+            res.data = docs;
+            res.meta = meta || {};
+            // store the query_type for subsequent middleware
+            res.meta.query_type = renderedQuery.type;
+
+            const messageParts = [
+              '[controller:search]',
+              `[queryType:${renderedQuery.type}]`,
+              `[es_result_count:${docs.length}]`
+            ];
+
+            logger.info(messageParts.join(' '));
+
+          }
+
+        }
+        logger.debug('[ES response]', docs);
+        next();
+      });
+
+    });
+
+  }
+
+  return controller;
+}
+
+module.exports = setup;

helper/debug.js

@@ -0,0 +1,43 @@
+'use strict';
+const _ = require('lodash');
+
+class Debug {
+  constructor(moduleName){
+    this.name = moduleName || 'unnamed module';
+  }
+
+  push(req, debugMsg){
+    if (!_.isEmpty(req.clean) && req.clean.enableDebug){
+      req.debug = req.debug || [];
+      // remove the extra space character
+      req.debug.push({[this.name]: debugMsg});
+    //  req.debug.push(`[${this.name}] ${debugMsg}`);
+    }
+  }
+  // optional debugMsg passed to timer
+  beginTimer(req, debugMsg){
+    if (!_.isEmpty(req.clean) && req.clean.enableDebug){
+      // internal object debugTimers. Doesn't get displayed in geocodeJSON
+      req.debugTimers = req.debugTimers || {};
+      req.debugTimers[this.name] = Date.now();
+      if (debugMsg){
+        this.push(req, `Timer Began: ${debugMsg}`);
+      } else {
+        this.push(req, `Timer Began`);
+      }
+    }
+  }
+
+  stopTimer(req, debugMsg){
+    if (!_.isEmpty(req.clean) && req.clean.enableDebug){
+      let timeElapsed = Date.now() - req.debugTimers[this.name];
+      if (debugMsg){
+        this.push(req, `Timer Stopped: ${timeElapsed} ms: ${debugMsg}`);
+      } else {
+        this.push(req, `Timer Stopped: ${timeElapsed} ms`);
+      }
+    }
+  }
+}
+
+module.exports = Debug;

middleware/changeLanguage.js

@@ -1,6 +1,5 @@
 
 var logger = require( 'pelias-logger' ).get( 'api' );
-var service = require('../service/language');
 const _ = require('lodash');
 
 /**
@@ -28,84 +27,34 @@ example response from language web service:
 }
 **/
 
-function setup() {
+function setup(service, should_execute) {
-  var transport = service.findById();
+  return function controller(req, res, next) {
-  var middleware = function(req, res, next) {
+    if (!should_execute(req, res)) {
-
-    // no-op, request did not require a language change
-    if( !isLanguageChangeRequired( req, res ) ){
       return next();
     }
 
-    // collect a list of parent ids to fetch translations for
+    service(req, res, (err, translations) => {
-    var ids = extractIds( res );
+      // if there's an error, log it and bail
-
-    // perform language lookup for all relevant ids
-    var timer = (new Date()).getTime();
-    transport.query( ids, function( err, translations ){
-
-      // update documents using a translation map
       if (err) {
-        logger.error( '[language] [error]', err );
+        logger.info(`[middleware:language][error]`);
-      } else {
+        logger.error(err);
-        updateDocs( req, res, translations );
+        return next();
       }
 
-      logger.info( '[language] [took]', (new Date()).getTime() - timer, 'ms' );
+      // otherwise, update all the docs with translations
+      updateDocs(req, res, _.defaultTo(translations, []));
       next();
-    });
-  };
-
-  middleware.transport = transport;
-  return middleware;
-}
-
-// collect a list of parent ids to fetch translations for
-function extractIds( res ){
-
-  // store ids in an object in order to avoid duplicates
-  var ids = {};
-
-  // convenience function for adding a new id to the object
-  function addId(id) {
-    ids[id] = true;
-  }
-
-  // extract all parent ids from documents
-  res.data.forEach( function( doc ){
-
-    // skip invalid records
-    if( !doc || !doc.parent ){ return; }
 
-    // iterate over doc.parent.* attributes
-    for( var attr in doc.parent ){
-
-      // match only attributes ending with '_id'
-      var match = attr.match(/_id$/);
-      if( !match ){ continue; }
-
-      // skip invalid/empty arrays
-      if( !Array.isArray( doc.parent[attr] ) || !doc.parent[attr].length ){
-        continue;
-      }
-
-      // add each id as a key in the ids object
-      doc.parent[attr].forEach( addId );
-    }
     });
 
-  // return a deduplicated array of ids
+  };
-  return Object.keys( ids );
+
 }
 
 // update documents using a translation map
 function updateDocs( req, res, translations ){
-
-  // sanity check arguments
-  if( !req || !res || !res.data || !translations ){ return; }
-
   // this is the target language we will be translating to
-  var requestLanguage = req.language.iso6393;
+  var requestLanguage = req.clean.lang.iso6393;
 
   // iterate over response documents
   res.data.forEach( function( doc, p ){
@@ -136,17 +85,14 @@ function updateDocs( req, res, translations ){
         if( !id ){ continue; }
 
         // id not found in translation service response
-        if( !translations.hasOwnProperty( id ) ){
+        if( !_.has(translations, id)){
-          logger.error( '[language] [error]', 'failed to find translations for', id );
+          logger.debug( `[language] [debug] failed to find translations for ${id}` );
           continue;
         }
 
-        // skip invalid records
-        if( !translations[id].hasOwnProperty( 'names' ) ){ continue; }
-
         // requested language is not available
         if (_.isEmpty(_.get(translations[id].names, requestLanguage, [] ))) {
-          logger.debug( '[language] [debug]', 'missing translation', requestLanguage, id );
+          logger.debug( `[language] [debug] missing translation ${requestLanguage} ${id}` );
           continue;
         }
 

middleware/geocodeJSON.js

@@ -64,6 +64,7 @@ function convertToGeocodeJSON(req, res, next, opts) {
   // OPTIONAL. Warnings and errors.
   addMessages(req, 'warnings', res.body.geocoding);
   addMessages(req, 'errors', res.body.geocoding);
+  addMessages(req, 'debug', res.body.geocoding);
 
   // OPTIONAL
   // Freeform

middleware/interpolate.js

@@ -1,7 +1,7 @@
-
+const async = require('async');
-var async = require('async');
+const logger = require( 'pelias-logger' ).get( 'api' );
-var logger = require( 'pelias-logger' ).get( 'api' );
+const source_mapping = require('../helper/type_mapping').source_mapping;
-var service = require('../service/interpolation');
+const _ = require('lodash');
 
 /**
 example response from interpolation web service:
@@ -21,132 +21,96 @@ example response from interpolation web service:
 }
 **/
 
-function setup() {
+function setup(service, should_execute) {
-
+  return function controller(req, res, next) {
-  var transport = service.search();
+    if (!should_execute(req, res)) {
-  var middleware = function(req, res, next) {
-
-    // no-op, user did not request an address
-    if( !isAddressQuery( req ) ){
       return next();
     }
 
-    // bind parsed_text variables to function call
+    // bind the service to the req which doesn't change
-    var bound = interpolate.bind( transport, req.clean.parsed_text );
+    const req_bound_service = _.partial(service, req);
-
-    // perform interpolations asynchronously for all relevant hits
-    var timer = (new Date()).getTime();
-    async.map( res.data, bound, function( err, results ){
-
-      // update res.data with the mapped values
-      if( !err ){
-        res.data = results;
-      }
-
-      // sort the results to ensure that addresses show up higher than street centroids      
-      res.data = res.data.sort((a, b) => {
-        if (a.layer === 'address' && b.layer !== 'address') { return -1; }
-        if (a.layer !== 'address' && b.layer === 'address') { return 1; }
-        return 0;
-      });
-
-      // log the execution time, continue
-      logger.info( '[interpolation] [took]', (new Date()).getTime() - timer, 'ms' );
-      next();
-    });
-  };
-
-  middleware.transport = transport;
-  return middleware;
-}
-
-function interpolate( parsed_text, hit, cb ){
 
-  // no-op, this hit is not from the 'street' layer
+    // only interpolate the street-layer results
-  // note: no network request is performed.
+    // save this off into a separate array so that when docs are annotated
-  if( !hit || hit.layer !== 'street' ){
+    //  after the interpolate results are returned, no complicated bookkeeping is needed
-    return cb( null, hit );
+    const street_results = _.get(res, 'data', []).filter(result => result.layer === 'street');
-  }
-
-  // query variables
-  var coord = hit.center_point;
-  var number = parsed_text.number;
-  var street = hit.address_parts.street || parsed_text.street;
-
-  // query interpolation service
-  this.query( coord, number, street, function( err, data ){
 
-    // an error occurred
+    // perform interpolations asynchronously for all relevant hits
-    // note: leave this hit unmodified
+    const start = (new Date()).getTime();
+    async.map(street_results, req_bound_service, (err, interpolation_results) => {
       if (err) {
-      logger.error( '[interpolation] [error]', err );
+        logger.error(`[middleware:interpolation] ${_.defaultTo(err.message, err)}`);
-      return cb( null, hit );
+        return next();
       }
 
-    // invalid / not useful response
+      interpolation_results.forEach((interpolation_result, idx) => {
+        const source_result = street_results[idx];
+
+        // invalid / not useful response, debug log for posterity
         // note: leave this hit unmodified
-    if( !data || !data.hasOwnProperty('properties') ){
+        if (!_.has(interpolation_result, 'properties')) {
-      logger.info( '[interpolation] [miss]', parsed_text );
+          logger.debug(`[interpolation] [miss] ${req.clean.parsed_text}`);
-      return cb( null, hit );
+          return;
         }
 
-    // the interpolation service returned a valid result
+        // the interpolation service returned a valid result, debug log for posterity
-    // note: we now merge thos values with the existing 'street' record.
+        // note: we now merge those values with the existing 'street' record
-    logger.info( '[interpolation] [hit]', parsed_text, data );
+        logger.debug(`[interpolation] [hit] ${req.clean.parsed_text} ${JSON.stringify(interpolation_result)}`);
-
-    // safety first!
-    try {
 
-      // -- metatdata --
+        // -- metadata --
-      hit.layer = 'address';
+        source_result.layer = 'address';
-      hit.match_type = 'interpolated';
+        source_result.match_type = 'interpolated';
 
         // -- name --
-      hit.name.default = data.properties.number + ' ' + hit.name.default;
+        source_result.name.default = `${interpolation_result.properties.number} ${source_result.name.default}`;
 
         // -- source --
-      var source = 'mixed';
+        // lookup the lowercased source, defaulting to 'mixed' when not found
-      if( data.properties.source === 'OSM' ){ source = 'openstreetmap'; }
+        // the source mapping is a jagged string->array, so default to 'mixed' as an array
-      else if( data.properties.source === 'OA' ){ source = 'openaddresses'; }
+        //  to ensure that subscript works
-      hit.source = source;
+        source_result.source = _.defaultTo(
+          source_mapping[_.toLower(interpolation_result.properties.source)],
+          ['mixed']
+        )[0];
 
         // -- source_id --
         // note: interpolated values have no source_id
-      delete hit.source_id; // remove original street source_id
+        delete source_result.source_id; // remove original street source_id
-      if( data.properties.hasOwnProperty( 'source_id' ) ){
+        if( interpolation_result.properties.hasOwnProperty( 'source_id' ) ){
-        hit.source_id = data.properties.source_id;
+          source_result.source_id = interpolation_result.properties.source_id;
         }
 
         // -- address_parts --
-      hit.address_parts.number = data.properties.number;
+        source_result.address_parts.number = interpolation_result.properties.number;
 
         // -- geo --
-      hit.center_point = {
+        source_result.center_point = {
-        lat: data.properties.lat,
+          lat: interpolation_result.properties.lat,
-        lon: data.properties.lon
+          lon: interpolation_result.properties.lon
         };
 
         // -- bbox --
-      delete hit.bounding_box;
+        delete source_result.bounding_box;
 
-      // return the modified hit
+      });
-      return cb( null, hit );
 
-    // a syntax error occurred in the code above (this shouldn't happen!)
+      // sort the results to ensure that addresses show up higher than street centroids
-    // note: the hit object may be partially modified, could possibly be invalid
+      if (_.has(res, 'data')) {
-    } catch( e ){
+        res.data.sort((a, b) => {
-      logger.error( '[interpolation] [error]', e, e.stack );
+          if (a.layer === 'address' && b.layer !== 'address') { return -1; }
-      return cb( null, hit );
+          if (a.layer !== 'address' && b.layer === 'address') { return 1; }
-    }
+          return 0;
         });
       }
 
-// boolean function to check if an address was requested by the user
+      // log the execution time, continue
-function isAddressQuery( req ){
+      logger.info( `[interpolation] [took] ${(new Date()).getTime() - start} ms`);
-  return req && req.hasOwnProperty('clean') &&
+      next();
-         req.clean.hasOwnProperty('parsed_text') &&
+
-         req.clean.parsed_text.hasOwnProperty('number') &&
+    });
-         req.clean.parsed_text.hasOwnProperty('street');
+
+  };
+
 }
 
 module.exports = setup;

package.json

@@ -40,11 +40,11 @@
     "addressit": "1.5.0",
     "async": "^2.0.0",
     "check-types": "^7.0.0",
-    "elasticsearch": "^12.0.1",
+    "elasticsearch": "^13.0.0",
     "elasticsearch-exceptions": "0.0.4",
     "express": "^4.8.8",
     "extend": "^3.0.1",
-    "geojson": "^0.4.0",
+    "geojson": "^0.5.0",
     "@mapbox/geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "iso-639-3": "^1.0.0",
@@ -55,18 +55,17 @@
     "markdown": "0.5.0",
     "morgan": "^1.8.2",
     "pelias-categories": "1.2.0",
-    "pelias-config": "2.11.0",
+    "pelias-config": "2.12.0",
     "pelias-labels": "1.6.0",
     "pelias-logger": "0.2.0",
-    "pelias-microservice-wrapper": "1.1.3",
+    "pelias-microservice-wrapper": "1.2.0",
-    "pelias-model": "5.0.0",
+    "pelias-model": "5.0.1",
-    "pelias-query": "8.16.1",
+    "pelias-query": "9.0.0",
     "pelias-sorting": "1.0.1",
-    "pelias-text-analyzer": "1.8.3",
+    "pelias-text-analyzer": "1.9.1",
     "predicates": "^1.0.1",
     "retry": "^0.10.1",
     "stats-lite": "^2.0.4",
-    "superagent": "^3.2.1",
     "through2": "^2.0.3"
   },
   "devDependencies": {
@@ -79,11 +78,11 @@
     "pelias-mock-logger": "1.1.1",
     "precommit-hook": "^3.0.0",
     "proxyquire": "^1.7.10",
-    "semantic-release": "^6.3.2",
+    "semantic-release": "^7.0.1",
     "source-map": "^0.5.6",
     "tap-dot": "1.0.5",
     "tape": "^4.5.1",
-    "tmp": "0.0.31",
+    "tmp": "0.0.33",
     "uglify-js": "^3.0.4"
   },
   "pre-commit": [

query/address_search_using_ids.js

@@ -0,0 +1,192 @@
+const peliasQuery = require('pelias-query');
+const defaults = require('./search_defaults');
+const logger = require('pelias-logger').get('api');
+const _ = require('lodash');
+const check = require('check-types');
+
+//------------------------------
+// general-purpose search query
+//------------------------------
+const addressUsingIdsQuery = new peliasQuery.layout.AddressesUsingIdsQuery();
+
+// scoring boost
+addressUsingIdsQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
+// --------------------------------
+
+// non-scoring hard filters
+addressUsingIdsQuery.filter( peliasQuery.view.boundary_country );
+addressUsingIdsQuery.filter( peliasQuery.view.boundary_circle );
+addressUsingIdsQuery.filter( peliasQuery.view.boundary_rect );
+addressUsingIdsQuery.filter( peliasQuery.view.sources );
+// --------------------------------
+
+// This query is a departure from traditional Pelias queries where textual
+// names of admin areas were looked up.  This query uses the ids returned by
+// placeholder for lookups which dramatically reduces the amount of information
+// that ES has to store and allows us to have placeholder handle altnames on
+// behalf of Pelias.
+//
+// For the happy path, an input like '30 West 26th Street, Manhattan' would result
+// in:
+// neighbourhood_id in []
+// borough_id in [421205771]
+// locality_id in [85945171, 85940551, 85972655]
+// localadmin_id in [404502889, 404499147, 404502891, 85972655]
+//
+// Where the ids are for all the various Manhattans.  Each of those could
+// conceivably be the Manhattan that the user was referring to so so all must be
+// queried for at the same time.
+//
+// A counter example for this is '1 West Market Street, York, PA' where York, PA
+// can be interpreted as a locality OR county.  From experience, when there's
+// ambiguity between locality and county for an input, the user is, with complete
+// metaphysical certitude, referring to the city.  If they were referring to the
+// county, they would have entered 'York County, PA'.  The point is that it's
+// insufficient to just query for all ids because, in this case, '1 West Market Street'
+// in other cities in York County, PA would be returned and would be both jarring
+// to the user and almost certainly leads to incorrect results.  For example,
+// the following could be returned (all are towns in York County, PA):
+// - 1 West Market Street, Dallastown, PA
+// - 1 West Market Street, Fawn Grove, PA
+// - 1 West Market Street, Shrewsbury, PA
+// etc.
+//
+// To avoid this calamitous response, this query takes the approach of
+// "granularity bands".  That is, if there are any ids in the first set of any
+// of these granularities:
+// - neighbourhood
+// - borough
+// - locality
+// - localadmin
+// - region
+// - macroregion
+// - dependency
+// - country
+//
+// then query for all ids in only those layers.  Falling back, if there are
+// no ids in those layers, query for the county/macrocounty layers.
+//
+// This methodology ensures that no happened-to-match-on-county results are returned.
+//
+// The decision was made to include all other layers in one to solve the issue
+// where a country and city share a name, such as Mexico, which could be
+// interpreted as a country AND city (in Missouri).  The data itself will sort
+// out which is correct.  That is, it's unlikely that "11 Rock Springs Dr" exists
+// in Mexico the country due to naming conventions and would be filtered out
+// (though it could, but that's good because it's legitimate)
+
+const granularity_bands = [
+  ['neighbourhood', 'borough', 'locality', 'localadmin', 'region', 'macroregion', 'dependency', 'country'],
+  ['county', 'macrocounty']
+];
+
+// returns IFF there are *any* results in the granularity band
+function anyResultsAtGranularityBand(results, band) {
+  return results.some(result => _.includes(band, result.layer));
+}
+
+// returns the ids of results at the requested layer
+function getIdsAtLayer(results, layer) {
+  return results.filter(result => result.layer === layer).map(_.property('source_id'));
+}
+
+/**
+  map request variables to query variables for all inputs
+  provided by this HTTP request.  This function operates on res.data which is the
+  Document-ified placeholder repsonse.
+**/
+function generateQuery( clean, res ){
+  const vs = new peliasQuery.Vars( defaults );
+  const results = _.defaultTo(res.data, []);
+
+  const logParts = ['query:address_search_using_ids', 'parser:libpostal'];
+
+  // sources
+  if( !_.isEmpty(clean.sources) ) {
+    vs.var( 'sources', clean.sources);
+    logParts.push('param:sources');
+  }
+
+  // size
+  if( clean.querySize ) {
+    vs.var( 'size', clean.querySize );
+    logParts.push('param:querySize');
+  }
+
+  if( ! _.isEmpty(clean.parsed_text.number) ){
+    vs.var( 'input:housenumber', clean.parsed_text.number );
+  }
+  vs.var( 'input:street', clean.parsed_text.street );
+
+  // find the first granularity band for which there are results
+  const granularity_band = granularity_bands.find(band => anyResultsAtGranularityBand(results, band));
+
+  // if there's a granularity band, accumulate the ids from each layer in the band
+  // into an object mapping layer->ids of those layers
+  if (granularity_band) {
+    const layers_to_ids = granularity_band.reduce((acc, layer) => {
+      acc[layer] = getIdsAtLayer(res.data, layer);
+      return acc;
+    }, {});
+
+    // use an object here instead of calling `set` since that flattens out an
+    // object into key/value pairs and makes identifying layers harder in query module
+    vs.var('input:layers', layers_to_ids);
+
+  }
+
+  // focus point
+  if( check.number(clean['focus.point.lat']) &&
+      check.number(clean['focus.point.lon']) ){
+    vs.set({
+      'focus:point:lat': clean['focus.point.lat'],
+      'focus:point:lon': clean['focus.point.lon']
+    });
+  }
+
+  // boundary rect
+  if( check.number(clean['boundary.rect.min_lat']) &&
+      check.number(clean['boundary.rect.max_lat']) &&
+      check.number(clean['boundary.rect.min_lon']) &&
+      check.number(clean['boundary.rect.max_lon']) ){
+    vs.set({
+      'boundary:rect:top': clean['boundary.rect.max_lat'],
+      'boundary:rect:right': clean['boundary.rect.max_lon'],
+      'boundary:rect:bottom': clean['boundary.rect.min_lat'],
+      'boundary:rect:left': clean['boundary.rect.min_lon']
+    });
+  }
+
+  // boundary circle
+  if( check.number(clean['boundary.circle.lat']) &&
+      check.number(clean['boundary.circle.lon']) ){
+    vs.set({
+      'boundary:circle:lat': clean['boundary.circle.lat'],
+      'boundary:circle:lon': clean['boundary.circle.lon']
+    });
+
+    if( check.number(clean['boundary.circle.radius']) ){
+      vs.set({
+        'boundary:circle:radius': Math.round( clean['boundary.circle.radius'] ) + 'km'
+      });
+    }
+  }
+
+  // boundary country
+  if( check.string(clean['boundary.country']) ){
+    vs.set({
+      'boundary:country': clean['boundary.country']
+    });
+  }
+
+  // format the log parts into a single coherent string
+  logger.info(logParts.map(part => `[${part}]`).join(' '));
+
+  return {
+    type: 'fallback',
+    body: addressUsingIdsQuery.render(vs)
+  };
+
+}
+
+module.exports = generateQuery;

query/search.js

@@ -10,16 +10,11 @@ const logger = require('pelias-logger').get('api');
 // general-purpose search query
 //------------------------------
 var fallbackQuery = new peliasQuery.layout.FallbackQuery();
-var geodisambiguationQuery = new peliasQuery.layout.GeodisambiguationQuery();
 
 // scoring boost
 fallbackQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
 fallbackQuery.score( peliasQuery.view.popularity_only_function );
 fallbackQuery.score( peliasQuery.view.population_only_function );
-
-geodisambiguationQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
-geodisambiguationQuery.score( peliasQuery.view.popularity_only_function );
-geodisambiguationQuery.score( peliasQuery.view.population_only_function );
 // --------------------------------
 
 // non-scoring hard filters
@@ -29,13 +24,6 @@ fallbackQuery.filter( peliasQuery.view.boundary_rect );
 fallbackQuery.filter( peliasQuery.view.sources );
 fallbackQuery.filter( peliasQuery.view.layers );
 fallbackQuery.filter( peliasQuery.view.categories );
-
-geodisambiguationQuery.filter( peliasQuery.view.boundary_country );
-geodisambiguationQuery.filter( peliasQuery.view.boundary_circle );
-geodisambiguationQuery.filter( peliasQuery.view.boundary_rect );
-geodisambiguationQuery.filter( peliasQuery.view.sources );
-geodisambiguationQuery.filter( peliasQuery.view.layers );
-geodisambiguationQuery.filter( peliasQuery.view.categories );
 // --------------------------------
 
 /**
@@ -147,10 +135,7 @@ function getQuery(vs) {
 
   logger.info(`[query:search] [search_input_type:${determineQueryType(vs)}]`);
 
-  if (hasStreet(vs) ||
+  if (hasStreet(vs) || isPostalCodeOnly(vs)) {
-      isCityStateOnlyWithOptionalCountry(vs) ||
-      isCityCountryOnly(vs) ||
-      isPostalCodeOnly(vs)) {
     return {
       type: 'fallback',
       body: fallbackQuery.render(vs)
@@ -174,7 +159,8 @@ function determineQueryType(vs) {
     return 'venue';
   }
   else if (['neighbourhood', 'borough', 'postcode', 'county', 'region','country'].some(
-      (layer)=> { return vs.isset(`input:${layer}`);})) {
+    layer => vs.isset(`input:${layer}`)
+  )) {
     return 'admin';
   }
   return 'other';
@@ -184,37 +170,8 @@ function hasStreet(vs) {
   return vs.isset('input:street');
 }
 
-function isCityStateOnlyWithOptionalCountry(vs) {
-  var isSet = (layer) => {
-    return vs.isset(`input:${layer}`);
-  };
-
-  var allowedFields = ['locality', 'region'];
-  var disallowedFields = ['query', 'category', 'housenumber', 'street',
-                          'neighbourhood', 'borough', 'postcode', 'county'];
-
-  return allowedFields.every(isSet) && !disallowedFields.some(isSet);
-
-}
-
-function isCityCountryOnly(vs) {
-  var isSet = (layer) => {
-    return vs.isset(`input:${layer}`);
-  };
-
-  var allowedFields = ['locality', 'country'];
-  var disallowedFields = ['query', 'category', 'housenumber', 'street',
-                          'neighbourhood', 'borough', 'postcode', 'county', 'region'];
-
-  return allowedFields.every(isSet) &&
-        !disallowedFields.some(isSet);
-
-}
-
 function isPostalCodeOnly(vs) {
-  var isSet = (layer) => {
+  var isSet = layer => vs.isset(`input:${layer}`);
-    return vs.isset(`input:${layer}`);
-  };
 
   var allowedFields = ['postcode'];
   var disallowedFields = ['query', 'category', 'housenumber', 'street',

routes/v1.js

@@ -11,7 +11,7 @@ var sanitizers = {
   autocomplete: require('../sanitizer/autocomplete'),
   place: require('../sanitizer/place'),
   search: require('../sanitizer/search'),
-  search_fallback: require('../sanitizer/search_fallback'),
+  defer_to_addressit: require('../sanitizer/defer_to_addressit'),
   structured_geocoding: require('../sanitizer/structured_geocoding'),
   reverse: require('../sanitizer/reverse'),
   nearby: require('../sanitizer/nearby')
@@ -28,18 +28,21 @@ var middleware = {
 var controllers = {
   coarse_reverse: require('../controller/coarse_reverse'),
   mdToHTML: require('../controller/markdownToHtml'),
+  libpostal: require('../controller/libpostal'),
   place: require('../controller/place'),
   placeholder: require('../controller/placeholder'),
   search: require('../controller/search'),
+  search_with_ids: require('../controller/search_with_ids'),
   status: require('../controller/status')
 };
 
 var queries = {
-  libpostal: require('../query/search'),
+  cascading_fallback: require('../query/search'),
-  fallback_to_old_prod: require('../query/search_original'),
+  very_old_prod: require('../query/search_original'),
   structured_geocoding: require('../query/structured_geocoding'),
   reverse: require('../query/reverse'),
-  autocomplete: require('../query/autocomplete')
+  autocomplete: require('../query/autocomplete'),
+  address_using_ids: require('../query/address_search_using_ids')
 };
 
 /** ----------------------- controllers ----------------------- **/
@@ -71,14 +74,28 @@ const hasRequestErrors = require('../controller/predicates/has_request_errors');
 const isCoarseReverse = require('../controller/predicates/is_coarse_reverse');
 const isAdminOnlyAnalysis = require('../controller/predicates/is_admin_only_analysis');
 const hasResultsAtLayers = require('../controller/predicates/has_results_at_layers');
+const isAddressItParse = require('../controller/predicates/is_addressit_parse');
+const hasRequestCategories = require('../controller/predicates/has_request_parameter')('categories');
+const isOnlyNonAdminLayers = require('../controller/predicates/is_only_non_admin_layers');
+// this can probably be more generalized
+const isRequestSourcesOnlyWhosOnFirst = require('../controller/predicates/is_request_sources_only_whosonfirst');
+const hasRequestParameter = require('../controller/predicates/has_request_parameter');
+const hasParsedTextProperties = require('../controller/predicates/has_parsed_text_properties');
 
 // shorthand for standard early-exit conditions
 const hasResponseDataOrRequestErrors = any(hasResponseData, hasRequestErrors);
 const hasAdminOnlyResults = not(hasResultsAtLayers(['venue', 'address', 'street']));
 
+const hasNumberButNotStreet = all(
+  hasParsedTextProperties.any('number'),
+  not(hasParsedTextProperties.any('street'))
+);
+
 const serviceWrapper = require('pelias-microservice-wrapper').service;
 const PlaceHolder = require('../service/configurations/PlaceHolder');
 const PointInPolygon = require('../service/configurations/PointInPolygon');
+const Language = require('../service/configurations/Language');
+const Interpolation = require('../service/configurations/Interpolation');
 
 /**
  * Append routes to app
@@ -97,13 +114,116 @@ function addRoutes(app, peliasConfig) {
   const placeholderService = serviceWrapper(placeholderConfiguration);
   const isPlaceholderServiceEnabled = _.constant(placeholderConfiguration.isEnabled());
 
+  const changeLanguageConfiguration = new Language(_.defaultTo(peliasConfig.api.services.placeholder, {}));
+  const changeLanguageService = serviceWrapper(changeLanguageConfiguration);
+  const isChangeLanguageEnabled = _.constant(changeLanguageConfiguration.isEnabled());
+
+  const interpolationConfiguration = new Interpolation(_.defaultTo(peliasConfig.api.services.interpolation, {}));
+  const interpolationService = serviceWrapper(interpolationConfiguration);
+  const isInterpolationEnabled = _.constant(interpolationConfiguration.isEnabled());
+
   // fallback to coarse reverse when regular reverse didn't return anything
   const coarseReverseShouldExecute = all(
     isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData)
   );
 
-  const placeholderShouldExecute = all(
+  const libpostalShouldExecute = all(
-    not(hasResponseDataOrRequestErrors), isPlaceholderServiceEnabled, isAdminOnlyAnalysis
+    not(hasRequestErrors),
+    not(isRequestSourcesOnlyWhosOnFirst)
+  );
+
+  // execute placeholder if libpostal only parsed as admin-only and needs to
+  //  be geodisambiguated
+  const placeholderGeodisambiguationShouldExecute = all(
+    not(hasResponseDataOrRequestErrors),
+    isPlaceholderServiceEnabled,
+    // check request.clean for several conditions first
+    not(
+      any(
+        // layers only contains venue, address, or street
+        isOnlyNonAdminLayers,
+        // don't geodisambiguate if categories were requested
+        hasRequestCategories
+      )
+    ),
+    any(
+      // only geodisambiguate if libpostal returned only admin areas or libpostal was skipped
+      isAdminOnlyAnalysis,
+      isRequestSourcesOnlyWhosOnFirst
+    )
+  );
+
+  // execute placeholder if libpostal identified address parts but ids need to
+  //  be looked up for admin parts
+  const placeholderIdsLookupShouldExecute = all(
+    not(hasResponseDataOrRequestErrors),
+    isPlaceholderServiceEnabled,
+    // check clean.parsed_text for several conditions that must all be true
+    all(
+      // run placeholder if clean.parsed_text has 'street'
+      hasParsedTextProperties.any('street'),
+      // don't run placeholder if there's a query or category
+      not(hasParsedTextProperties.any('query', 'category')),
+      // run placeholder if there are any adminareas identified
+      hasParsedTextProperties.any('neighbourhood', 'borough', 'city', 'county', 'state', 'country')
+    )
+  );
+
+  const searchWithIdsShouldExecute = all(
+    not(hasRequestErrors),
+    // don't search-with-ids if there's a query or category
+    not(hasParsedTextProperties.any('query', 'category')),
+    // there must be a street
+    hasParsedTextProperties.any('street')
+  );
+
+  // placeholder should have executed, useful for determining whether to actually
+  //  fallback or not (don't fallback to old search if the placeholder response
+  //  should be honored as is)
+  const placeholderShouldHaveExecuted = any(
+    placeholderGeodisambiguationShouldExecute,
+    placeholderIdsLookupShouldExecute
+  );
+
+  // don't execute the cascading fallback query IF placeholder should have executed
+  //  that way, if placeholder didn't return anything, don't try to find more things the old way
+  const fallbackQueryShouldExecute = all(
+    not(hasRequestErrors),
+    not(hasResponseData),
+    not(placeholderShouldHaveExecuted)
+  );
+
+  // defer to addressit for analysis IF there's no response AND placeholder should not have executed
+  const shouldDeferToAddressIt = all(
+    not(hasRequestErrors),
+    not(hasResponseData),
+    not(placeholderShouldHaveExecuted)
+  );
+
+  // call very old prod query if addressit was the parser
+  const oldProdQueryShouldExecute = all(
+    not(hasRequestErrors),
+    isAddressItParse
+  );
+
+  // get language adjustments if:
+  // - there's a response
+  // - theres's a lang parameter in req.clean
+  const changeLanguageShouldExecute = all(
+    hasResponseData,
+    not(hasRequestErrors),
+    isChangeLanguageEnabled,
+    hasRequestParameter('lang')
+  );
+
+  // interpolate if:
+  // - there's a number and street
+  // - there are street-layer results (these are results that need to be interpolated)
+  const interpolationShouldExecute = all(
+    not(hasRequestErrors),
+    isInterpolationEnabled,
+    hasParsedTextProperties.all('number', 'street'),
+    hasResultsAtLayers('street')
   );
 
   // execute under the following conditions:
@@ -117,6 +237,10 @@ function addRoutes(app, peliasConfig) {
     )
   );
 
+  // helpers to replace vague booleans
+  const geometricFiltersApply = true;
+  const geometricFiltersDontApply = false;
+
   var base = '/v1/';
 
   /** ------------------------- routers ------------------------- **/
@@ -132,17 +256,20 @@ function addRoutes(app, peliasConfig) {
       sanitizers.search.middleware(peliasConfig.api),
       middleware.requestLanguage,
       middleware.calcSize(),
-      controllers.placeholder(placeholderService, placeholderShouldExecute),
+      controllers.libpostal(libpostalShouldExecute),
-      // 3rd parameter is which query module to use, use fallback/geodisambiguation
+      controllers.placeholder(placeholderService, geometricFiltersApply, placeholderGeodisambiguationShouldExecute),
-      //  first, then use original search strategy if first query didn't return anything
+      controllers.placeholder(placeholderService, geometricFiltersDontApply, placeholderIdsLookupShouldExecute),
-      controllers.search(peliasConfig.api, esclient, queries.libpostal, not(hasResponseDataOrRequestErrors)),
+      controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute),
-      sanitizers.search_fallback.middleware,
+      // 3rd parameter is which query module to use, use fallback first, then
-      controllers.search(peliasConfig.api, esclient, queries.fallback_to_old_prod, not(hasResponseDataOrRequestErrors)),
+      //  use original search strategy if first query didn't return anything
+      controllers.search(peliasConfig.api, esclient, queries.cascading_fallback, fallbackQueryShouldExecute),
+      sanitizers.defer_to_addressit(shouldDeferToAddressIt),
+      controllers.search(peliasConfig.api, esclient, queries.very_old_prod, oldProdQueryShouldExecute),
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig.api),
       postProc.confidenceScoresFallback(),
-      postProc.interpolate(),
+      postProc.interpolate(interpolationService, interpolationShouldExecute),
       postProc.sortResponseData(require('pelias-sorting'), hasAdminOnlyResults),
       postProc.dedupe(),
       postProc.accuracy(),
@@ -150,7 +277,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -164,14 +291,14 @@ function addRoutes(app, peliasConfig) {
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig.api),
       postProc.confidenceScoresFallback(),
-      postProc.interpolate(),
+      postProc.interpolate(interpolationService, interpolationShouldExecute),
       postProc.dedupe(),
       postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -188,7 +315,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -209,7 +336,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -229,7 +356,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON
@@ -243,7 +370,7 @@ function addRoutes(app, peliasConfig) {
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
       postProc.normalizeParentIds(),
-      postProc.changeLanguage(),
+      postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute),
       postProc.assignLabels(),
       postProc.geocodeJSON(peliasConfig.api, base),
       postProc.sendJSON

sanitizer/_boundary_country.js

@@ -1,7 +1,7 @@
-var check = require('check-types');
+const check = require('check-types');
-var iso3166 = require('iso3166-1');
+const iso3166 = require('iso3166-1');
 
-function sanitize(raw, clean) {
+function _sanitize(raw, clean) {
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
@@ -37,4 +37,11 @@ function containsIsoCode(isoCode) {
   return iso3166.is2(isoCode) || iso3166.is3(isoCode);
 }
 
-module.exports = sanitize;
+function _expected(){
+  return [{ name: 'boundary.country' }];
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_categories.js

@@ -1,4 +1,3 @@
-
 var check = require('check-types');
 var categoryTaxonomy = require('pelias-categories');
 
@@ -8,7 +7,7 @@ var ERRORS = {
 };
 
 // validate inputs, convert types and apply defaults
-function sanitize( raw, clean, categories ) {
+function _sanitize( raw, clean, categories ) {
 
   categories = categories || categoryTaxonomy;
 
@@ -50,5 +49,11 @@ function sanitize( raw, clean, categories ) {
   return messages;
 }
 
+function _expected() {
+  return [{ name: 'categories' }];
+}
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_city_name_standardizer.js

@@ -15,7 +15,7 @@ function transliterate(match) {
 }
 
 // transliterate ft/mt/saint/sainte to fort/mount/st/ste, respectively
-function sanitize(raw, clean) {
+function _sanitize(raw, clean) {
   // error & warning messages
   // this function doesn't add any error or warning messages
   const messages = { errors: [], warnings: [] };
@@ -44,4 +44,6 @@ function sanitize(raw, clean) {
 
 }
 
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_debug.js

@@ -0,0 +1,23 @@
+var _ = require('lodash');
+
+function _sanitize(raw, clean){
+  const messages = {errors: [], warnings: []};
+
+  if(!_.isUndefined(raw.debug) ){
+    clean.enableDebug = (typeof raw.debug === 'string') ? isTruthy(raw.debug.toLowerCase()) : isTruthy( raw.debug );
+  }
+  return messages;
+}
+
+function _expected() {
+  return [{ name: 'debug' }];
+}
+
+function isTruthy(val) {
+  return _.includes( ['true', '1', 1, true], val );
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_deprecate_quattroshapes.js

@@ -10,7 +10,7 @@ var _ = require('lodash');
   @see: https://github.com/pelias/api/issues/442
 **/
 
-function sanitize( raw, clean, opts ) {
+function _sanitize( raw, clean, opts ) {
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
@@ -38,4 +38,6 @@ function sanitize( raw, clean, opts ) {
   return messages;
 }
 
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_flag_bool.js

@@ -5,26 +5,28 @@ var _ = require('lodash');
  *
  * @param {string} paramName name of parameter being sanitized
  * @param {boolean} defaultValue value to set variable to if none specified
- * @returns {Function}
+ * @returns {Object} object containing functions
  */
-function setup( paramName, defaultValue ) {
+function _setup( paramName, defaultValue ) {
-  return function( raw, clean ){
+  /**
-    return sanitize( raw, clean, {
+   * {object} opts
+  */
+
+  const opts = {
     paramName: paramName,
     defaultValue: defaultValue
-    });
   };
-}
 
+  return {
     /**
      * Validate inputs, convert types and apply defaults
      *
      * @param {object} raw
      * @param {object} clean
- * @param {object} opts
      * @returns {{errors: Array, warnings: Array}}
      */
-function sanitize( raw, clean, opts ){
+
+    sanitize: function _sanitize( raw, clean){
 
       // error & warning messages`1
       var messages = { errors: [], warnings: [] };
@@ -36,7 +38,13 @@ function sanitize( raw, clean, opts ){
         clean[opts.paramName] = opts.defaultValue;
       }
       return messages;
-}
+    }, // end of _sanitize function
+
+    expected: function _expected(){
+      return [{ name: opts.paramName}];
+    } // end of _expected function
+  }; // end of return object
+} // end of _setup function
 
 /**
  * Determine if param value is "truthy"
@@ -47,4 +55,4 @@ function isTruthy(val) {
   return _.includes( ['true', '1', 1, true], val );
 }
 
-module.exports = setup;
+module.exports = _setup;

sanitizer/_geo_autocomplete.js

@@ -3,7 +3,7 @@ var LAT_LON_IS_REQUIRED = false;
 var RECT_IS_REQUIRED = false;
 
 // validate inputs, convert types and apply defaults
-module.exports = function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -17,4 +17,19 @@ module.exports = function sanitize( raw, clean ){
   }
 
   return messages;
-};
+}
+
+function _expected(){
+  return [
+  { name: 'focus.point.lat' },
+  { name: 'focus.point.lon' },
+  { name: 'boundary.rect.min_lat' },
+  { name: 'boundary.rect.max_lat' },
+  { name: 'boundary.rect.min_lon' },
+  { name: 'boundary.rect.max_lon' }];
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_geo_reverse.js

@@ -7,7 +7,7 @@ var LAT_LON_IS_REQUIRED = true,
 const non_coarse_layers = ['venue', 'address', 'street'];
 
 // validate inputs, convert types and apply defaults
-module.exports = function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -38,4 +38,18 @@ module.exports = function sanitize( raw, clean ){
   }
 
   return messages;
-};
+}
+
+function _expected(){
+  return [
+  { name: 'point.lat' },
+  { name: 'point.lon' },
+  { name: 'boundary.circle.lon' }, // copied from point.lon by the API, not user input
+  { name: 'boundary.circle.lat' }, // copied from point.lat by the API, not user input
+  { name: 'boundary.circle.radius'}];
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_geo_search.js

@@ -6,7 +6,7 @@ var RECT_IS_REQUIRED = false;
 var CIRCLE_IS_REQUIRED = false;
 
 // validate inputs, convert types and apply defaults
-module.exports = function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -21,4 +21,22 @@ module.exports = function sanitize( raw, clean ){
   }
 
   return messages;
-};
+}
+
+function _expected(){
+  return [
+    { name: 'focus.point.lat' },
+    { name: 'focus.point.lon' },
+    { name: 'boundary.circle.lon'},
+    { name: 'boundary.circle.lat'},
+    { name: 'boundary.circle.radius'},
+    { name: 'boundary.rect.min_lat' },
+    { name: 'boundary.rect.max_lat' },
+    { name: 'boundary.rect.min_lon' },
+    { name: 'boundary.rect.max_lon' }];
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_geonames_deprecation.js

@@ -5,7 +5,7 @@ with the release of coarse reverse as a separate thing and ES reverse only
 handling venues, addresses, and streets, geonames make no sense in the reverse context
 **/
 
-function sanitize( raw, clean, opts ) {
+function _sanitize( raw, clean, opts ) {
   // error & warning messages
   const messages = { errors: [], warnings: [] };
 
@@ -22,4 +22,6 @@ function sanitize( raw, clean, opts ) {
 
 }
 
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_geonames_warnings.js

@@ -9,7 +9,7 @@ function hasAnyNonAdminFields(parsed_text) {
               non_admin_fields));
 }
 
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
   // error & warning messages
   const messages = { errors: [], warnings: [] };
 
@@ -34,4 +34,6 @@ function sanitize( raw, clean ){
   return messages;
 }
 
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_ids.js

@@ -52,7 +52,7 @@ function sanitizeId(rawId, messages) {
   };
 }
 
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
@@ -84,5 +84,11 @@ function sanitize( raw, clean ){
   return messages;
 }
 
+function _expected(){
+  return [{ name: 'ids' }];
+}
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_iso2_to_iso3.js

@@ -4,7 +4,7 @@ const iso3166 = require('iso3166-1');
 // this sanitizer exists solely to convert an ISO2 country value to ISO3
 // eg - 'TH' -> 'THA'
 // this can go away once altnames imports ISO2 country values from WOF
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
   // error & warning messages
   const messages = { errors: [], warnings: [] };
 
@@ -16,4 +16,6 @@ function sanitize( raw, clean ){
 }
 
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_location_bias.js

@@ -4,9 +4,9 @@ Set a focus.lat and focus.lon if specified in pelias config
 * @param {object} defaultParameters property of pelias config
 */
 
-function setup(defaultParameters){
+function _setup(defaultParameters){
-
+  return {
-  return function setLocationBias(raw, clean){
+    sanitize: function sanitize(raw, clean){
   /*
   check that:
   1. {object} raw exists
@@ -24,8 +24,9 @@ check that:
       }
 
       return { errors: [], warnings: [] };
+    }
   };
 }
 
 // if focus.point.lat and focus.point.lon already exists, don't change
-module.exports = setup;
+module.exports = _setup;

sanitizer/_single_scalar_parameters.js

@@ -3,7 +3,7 @@ var _ = require('lodash'),
     check = require('check-types');
 
 // validate inputs
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
@@ -22,4 +22,6 @@ function sanitize( raw, clean ){
 }
 
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/_size.js

@@ -5,14 +5,15 @@ var MIN_SIZE = 1,
     DEFAULT_SIZE = 10;
 
 // validate inputs, convert types and apply defaults
-function setup( size_min, size_max, size_def ){
+function _setup( size_min, size_max, size_def ){
 
   // allow caller to inject custom min/max/default values
   if( !check.number( size_min ) ){ size_min = MIN_SIZE; }
   if( !check.number( size_max ) ){ size_max = MAX_SIZE; }
   if( !check.number( size_def ) ){ size_def = DEFAULT_SIZE; }
 
-  return function sanitize( raw, clean ){
+  return {
+    sanitize: function _sanitize( raw, clean ){
 
       // error & warning messages
       var messages = { errors: [], warnings: [] };
@@ -37,8 +38,14 @@ function setup( size_min, size_max, size_def ){
       }
 
       return messages;
+    },
+
+    expected: function _expected() {
+      // add size as a valid parameter
+      return [{ name: 'size' }];
+    }
   };
 }
 
 // export function
-module.exports = setup;
+module.exports = _setup;

sanitizer/_sources_and_layers.js

@@ -5,7 +5,7 @@ var type_mapping = require( '../helper/type_mapping' );
  * This sanitizer depends on clean.layers and clean.sources
  * so it has to be run after those sanitizers have been run
  */
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
   var messages = { errors: [], warnings: [] };
 
   var possible_errors = [];
@@ -34,4 +34,11 @@ function sanitize( raw, clean ){
   return messages;
 }
 
-module.exports = sanitize;
+function _expected(){
+  return [{ 'name': 'sources' }, { 'name': 'layers' }];
+}
+
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_synthesize_analysis.js

@@ -35,7 +35,7 @@ function getHouseNumberField(analyzed_address) {
 
 }
 
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   const messages = { errors: [], warnings: [] };
@@ -86,5 +86,20 @@ function sanitize( raw, clean ){
   return messages;
 }
 
+function _expected() {
+  return [
+    { 'name': 'venue' },
+    { 'name': 'address' },
+    { 'name': 'neighbourhood' },
+    { 'name': 'borough' },
+    { 'name': 'locality' },
+    { 'name': 'county' },
+    { 'name': 'region' },
+    { 'name': 'postalcode' },
+    { 'name': 'country' }];
+}
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_targets.js

@@ -5,17 +5,15 @@ function getValidKeys(mapping) {
   return _.uniq(Object.keys(mapping)).join(',');
 }
 
-function setup( paramName, targetMap ) {
+function _setup( paramName, targetMap ) {
-  return function( raw, clean ){
+  const opts = {
-    return sanitize( raw, clean, {
     paramName: paramName,
     targetMap: targetMap,
     targetMapKeysString: getValidKeys(targetMap)
-    });
   };
-}
 
-function sanitize( raw, clean, opts ) {
+  return {
+    sanitize: function _sanitize( raw, clean ) {
       // error & warning messages
       var messages = { errors: [], warnings: [] };
 
@@ -69,6 +67,10 @@ function sanitize( raw, clean, opts ) {
 
 
       return messages;
-}
+    } // end of _sanitize function
+
+  }; // end of object to be returned
+} // end of _setup function
+
 
-module.exports = setup;
+module.exports = _setup;

sanitizer/_text.js

@@ -1,33 +1,31 @@
-var check = require('check-types'),
+const check = require('check-types');
-  text_analyzer = require('pelias-text-analyzer');
+const _ = require('lodash');
 
 // validate texts, convert types and apply defaults
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
-  var messages = { errors: [], warnings: [] };
+  const messages = { errors: [], warnings: [] };
 
   // invalid input 'text'
   // must call `!check.nonEmptyString` since `check.emptyString` returns
   //  `false` for `undefined` and `null`
   if( !check.nonEmptyString( raw.text ) ){
     messages.errors.push('invalid param \'text\': text length, must be >0');
-  }
 
-  // valid input 'text'
+  } else {
-  else {
-    // valid text
     clean.text = raw.text;
 
-    // parse text with query parser
-    var parsed_text = text_analyzer.parse(clean.text);
-    if (check.assigned(parsed_text)) {
-      clean.parsed_text = parsed_text;
-    }
   }
 
   return messages;
 }
 
+function _expected(){
+  return [{ name: 'text' }];
+}
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});

sanitizer/_text_addressit.js

@@ -5,7 +5,7 @@ var _      = require('lodash');
 var logger = require('pelias-logger').get('api');
 
 // validate texts, convert types and apply defaults
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -20,6 +20,7 @@ function sanitize( raw, clean ){
 
     // valid text
     clean.text = raw.text;
+    clean.parser = 'addressit';
 
     // remove anything that may have been parsed before
     delete clean.parsed_text;
@@ -34,10 +35,15 @@ function sanitize( raw, clean ){
   return messages;
 }
 
-// export function
+function _expected(){
-module.exports = sanitize;
+  return [{ name: 'text' }];
-
+}
 
+// export function
+module.exports = () => ({
+  sanitize: _sanitize,
+  expected: _expected
+});
 
 // this is the addressit functionality from https://github.com/pelias/text-analyzer/blob/master/src/addressItParser.js
 var DELIM = ',';

sanitizer/_tokenizer.js

@@ -14,7 +14,7 @@ var check = require('check-types');
   note: this sanitizer should run *after* the '_text' sanitizer so it can
   use the output of clean.parsed_text where available.
 **/
-function sanitize( raw, clean ){
+function _sanitize( raw, clean ){
 
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -103,4 +103,6 @@ function sanitize( raw, clean ){
 }
 
 // export function
-module.exports = sanitize;
+module.exports = () => ({
+  sanitize: _sanitize
+});

sanitizer/autocomplete.js

@@ -4,30 +4,24 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 // middleware
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
-      singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      text: require('../sanitizer/_text_addressit'),
+      debug: require('../sanitizer/_debug')(),
-      tokenizer: require('../sanitizer/_tokenizer'),
+      text: require('../sanitizer/_text_addressit')(),
+      tokenizer: require('../sanitizer/_tokenizer')(),
       size: require('../sanitizer/_size')(10, 10, 10),
       layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitizers, must be run after them
-      sources_and_layers: require('../sanitizer/_sources_and_layers'),
+      sources_and_layers: require('../sanitizer/_sources_and_layers')(),
       private: require('../sanitizer/_flag_bool')('private', false),
       location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
-      geo_autocomplete: require('../sanitizer/_geo_autocomplete'),
+      geo_autocomplete: require('../sanitizer/_geo_autocomplete')(),
-      boundary_country: require('../sanitizer/_boundary_country'),
+      boundary_country: require('../sanitizer/_boundary_country')(),
-      categories: require('../sanitizer/_categories')
+      categories: require('../sanitizer/_categories')()
     };
 
-  var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+  return ( req, res, next ) => {
-
+    sanitizeAll.runAllChecks(req, sanitizers);
-  return function( req, res, next ){
-    sanitize( req, function( err, clean ){
-      if( err ){
-        res.status(400); // 400 Bad Request
-        return next(err);
-      }
     next();
-    });
   };
 };

sanitizer/defer_to_addressit.js

@@ -0,0 +1,31 @@
+const sanitizeAll = require('../sanitizer/sanitizeAll'),
+    sanitizers = {
+      debug: require('../sanitizer/_debug')(),
+      text: require('../sanitizer/_text_addressit')()
+    };
+
+const logger = require('pelias-logger').get('api');
+const logging = require( '../helper/logging' );
+
+// middleware
+module.exports = (should_execute) => {
+  return function(req, res, next) {
+    // if res.data already has results then don't call the _text_autocomplete sanitizer
+    // this has been put into place for when the libpostal integration way of querying
+    // ES doesn't return anything and we want to fallback to the old logic
+    if (!should_execute(req, res)) {
+      return next();
+    }
+
+    // log the query that caused a fallback since libpostal+new-queries didn't return anything
+    if (req.path === '/v1/search') {
+      const queryText = logging.isDNT(req) ? '[text removed]' : req.clean.text;
+      logger.info(`fallback queryText: ${queryText}`);
+    }
+
+    sanitizeAll.sanitize(req, sanitizers);
+    next();
+
+  };
+
+};

sanitizer/nearby.js

@@ -1,21 +1,25 @@
-var _ = require('lodash');
 var sanitizeAll = require('../sanitizer/sanitizeAll');
-var reverseSanitizers = require('./reverse').sanitizer_list;
+var type_mapping = require('../helper/type_mapping');
 
 // add categories to the sanitizer list
-var sanitizers = _.merge({}, reverseSanitizers, {
+var sanitizers = {
-  categories: require('../sanitizer/_categories')
+  singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-});
+  debug: require('../sanitizer/_debug')(),
-
+  quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
-var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+  layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
-
+  sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
-// export sanitize for testing
+  // depends on the layers and sources sanitizers, must be run after them
-module.exports.sanitize = sanitize;
+  sources_and_layers: require('../sanitizer/_sources_and_layers')(),
-module.exports.sanitizer_list = sanitizers;
+  geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
+  size: require('../sanitizer/_size')(/* use defaults*/),
+  private: require('../sanitizer/_flag_bool')('private', false),
+  geo_reverse: require('../sanitizer/_geo_reverse')(),
+  boundary_country: require('../sanitizer/_boundary_country')(),
+  categories: require('../sanitizer/_categories')()
+};
 
 // middleware
 module.exports.middleware = function( req, res, next ){
-  sanitize( req, function( err, clean ){
+  sanitizeAll.runAllChecks(req, sanitizers);
   next();
-  });
 };

sanitizer/place.js

@@ -1,20 +1,14 @@
 
 var sanitizeAll = require('../sanitizer/sanitizeAll'),
     sanitizers = {
-      singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      ids: require('../sanitizer/_ids'),
+      debug: require('../sanitizer/_debug')(),
+      ids: require('../sanitizer/_ids')(),
       private: require('../sanitizer/_flag_bool')('private', false)
     };
 
-var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
-
-// export sanitize for testing
-module.exports.sanitize = sanitize;
-module.exports.sanitizer_list = sanitizers;
-
 // middleware
 module.exports.middleware = function(req, res, next){
-  sanitize( req, function( err, clean ){
+  sanitizeAll.runAllChecks(req, sanitizers);
   next();
-  });
 };

sanitizer/reverse.js

@@ -2,28 +2,22 @@
 var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitizer/sanitizeAll'),
     sanitizers = {
-      singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
+      debug: require('../sanitizer/_debug')(),
+      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
       layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitizers, must be run after them
-      sources_and_layers: require('../sanitizer/_sources_and_layers'),
+      sources_and_layers: require('../sanitizer/_sources_and_layers')(),
-      geonames_deprecation: require('../sanitizer/_geonames_deprecation'),
+      geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
       size: require('../sanitizer/_size')(/* use defaults*/),
       private: require('../sanitizer/_flag_bool')('private', false),
-      geo_reverse: require('../sanitizer/_geo_reverse'),
+      geo_reverse: require('../sanitizer/_geo_reverse')(),
-      boundary_country: require('../sanitizer/_boundary_country')
+      boundary_country: require('../sanitizer/_boundary_country')()
     };
 
-var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
-
-// export sanitize for testing
-module.exports.sanitize = sanitize;
-module.exports.sanitizer_list = sanitizers;
-
 // middleware
 module.exports.middleware = function( req, res, next ){
-  sanitize( req, function( err, clean ){
+  sanitizeAll.runAllChecks(req, sanitizers);
   next();
-  });
 };

sanitizer/sanitizeAll.js

@@ -1,4 +1,5 @@
-function sanitize( req, sanitizers, cb ){
+'use strict';
+function sanitize( req, sanitizers ){
   // init an object to store clean (sanitized) input parameters if not initialized
   req.clean = req.clean || {};
 
@@ -8,10 +9,10 @@ function sanitize( req, sanitizers, cb ){
 
   // source of input parameters
   // (in this case from the GET querystring params)
-  var params = req.query || {};
+  const params = req.query || {};
 
-  for (var s in sanitizers) {
+  for (let s in sanitizers) {
-    var sanity = sanitizers[s]( params, req.clean );
+    var sanity = sanitizers[s].sanitize( params, req.clean );
 
     // if errors occurred then set them
     // on the req object.
@@ -25,10 +26,51 @@ function sanitize( req, sanitizers, cb ){
       req.warnings = req.warnings.concat( sanity.warnings );
     }
   }
+}
+
+// Adds to goodParameters every acceptable parameter passed through API call
+function checkParameters( req, sanitizers ) {
+  req.warnings = req.warnings || [];
+  // source of input parameters
+  // (in this case from the GET querystring params)
+  const params = req.query || {};
+  const goodParameters = {};
+
+  for (let s in sanitizers) {
+
+    // checks if function exists
+    if (typeof sanitizers[s].expected === 'function'){
+      /** expected() returns {array} ex: [{ name: 'text' }] */
+      for (let t in sanitizers[s].expected()) {
+        /** {object} prop */
+        const prop = sanitizers[s].expected()[t];
+        if (prop.hasOwnProperty('name')){
+          // adds name of valid parameter
+          goodParameters[prop.name] = prop.name;
+        }
+      }
+    }
+  }
+  // If there are any unexpected parameters & goodParameters isn't empty,
+  // add a warning message
+  if (Object.keys(goodParameters).length !== 0) {
+    for (let p in params) {
+      if (!goodParameters.hasOwnProperty(p)){
+        req.warnings = req.warnings.concat('Invalid Parameter: ' + p);
+      }
+    }
+  }
+}
 
-  // @todo remove these args, they do not need to be passed out
+// runs both sanitize and checkParameters functions in async parallel
-  return cb( undefined, req.clean );
+function runAllChecks (req, sanitizers) {
+  sanitize(req, sanitizers);
+  checkParameters(req, sanitizers);
 }
 
 // export function
-module.exports = sanitize;
+module.exports = {
+  sanitize: sanitize,
+  checkParameters: checkParameters,
+  runAllChecks: runAllChecks
+};

sanitizer/search.js

@@ -3,31 +3,27 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 // middleware
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
-        singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+        singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-        quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
+        debug: require('../sanitizer/_debug')(),
-        text: require('../sanitizer/_text'),
+        quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
-        iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
+        text: require('../sanitizer/_text')(),
-        city_name_standardizer: require('../sanitizer/_city_name_standardizer'),
         size: require('../sanitizer/_size')(/* use defaults*/),
         layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
         sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
         // depends on the layers and sources sanitizers, must be run after them
-        sources_and_layers: require('../sanitizer/_sources_and_layers'),
+        sources_and_layers: require('../sanitizer/_sources_and_layers')(),
         private: require('../sanitizer/_flag_bool')('private', false),
         location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
-        geo_search: require('../sanitizer/_geo_search'),
+        geo_search: require('../sanitizer/_geo_search')(),
-        boundary_country: require('../sanitizer/_boundary_country'),
+        boundary_country: require('../sanitizer/_boundary_country')(),
-        categories: require('../sanitizer/_categories'),
+        categories: require('../sanitizer/_categories')(),
         // this can go away once geonames has been abrogated
-        geonames_warnings: require('../sanitizer/_geonames_warnings')
+        geonames_warnings: require('../sanitizer/_geonames_warnings')()
       };
 
-  var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+  return ( req, res, next ) => {
-
+    sanitizeAll.runAllChecks(req, sanitizers);
-  return function( req, res, next ){
-    sanitize( req, function( err, clean ){
     next();
-    });
   };
 
 };

sanitizer/search_fallback.js

@@ -1,30 +0,0 @@
-var sanitizeAll = require('../sanitizer/sanitizeAll'),
-    sanitizers = {
-      text: require('../sanitizer/_text_addressit')
-    };
-
-var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
-var logger = require('pelias-logger').get('api');
-var logging = require( '../helper/logging' );
-var _ = require('lodash');
-
-// middleware
-module.exports.middleware = function( req, res, next ){
-  // if res.data already has results then don't call the _text_autocomplete sanitizer
-  // this has been put into place for when the libpostal integration way of querying
-  // ES doesn't return anything and we want to fallback to the old logic
-  if (_.get(res, 'data', []).length > 0) {
-    return next();
-  }
-
-  // log the query that caused a fallback since libpostal+new-queries didn't return anything
-  if (req.path === '/v1/search') {
-    const queryText = logging.isDNT(req) ? '[text removed]' : req.clean.text;
-    logger.info(`fallback queryText: ${queryText}`);
-  }
-
-  sanitize( req, function( err, clean ){
-    next();
-  });
-
-};

sanitizer/structured_geocoding.js

@@ -4,27 +4,28 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 // middleware
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
-        singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+        singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-        quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
+        debug: require('../sanitizer/_debug')(),
-        synthesize_analysis: require('../sanitizer/_synthesize_analysis'),
+        quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
-        iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
+        synthesize_analysis: require('../sanitizer/_synthesize_analysis')(),
-        city_name_standardizer: require('../sanitizer/_city_name_standardizer'),
+        iso2_to_iso3: require('../sanitizer/_iso2_to_iso3')(),
+        city_name_standardizer: require('../sanitizer/_city_name_standardizer')(),
         size: require('../sanitizer/_size')(/* use defaults*/),
         layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
         sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
         // depends on the layers and sources sanitizers, must be run after them
-        sources_and_layers: require('../sanitizer/_sources_and_layers'),
+        sources_and_layers: require('../sanitizer/_sources_and_layers')(),
         private: require('../sanitizer/_flag_bool')('private', false),
         location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
-        geo_search: require('../sanitizer/_geo_search'),
+        geo_search: require('../sanitizer/_geo_search')(),
-        boundary_country: require('../sanitizer/_boundary_country'),
+        boundary_country: require('../sanitizer/_boundary_country')(),
-        categories: require('../sanitizer/_categories')
+        categories: require('../sanitizer/_categories')()
       };
-  var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
-  return function( req, res, next ){
+  return ( req, res, next ) => {
-    sanitize( req, function( err, clean ){
+    sanitizeAll.runAllChecks(req, sanitizers);
     next();
-    });
+
   };
+
 };

schema.js

@@ -36,6 +36,11 @@ module.exports = Joi.object().keys({
         url: Joi.string().uri({ scheme: /https?/ }),
         timeout: Joi.number().integer().optional().default(250).min(0),
         retries: Joi.number().integer().optional().default(3).min(0),
+      }).unknown(false).requiredKeys('url'),
+      interpolation: Joi.object().keys({
+        url: Joi.string().uri({ scheme: /https?/ }),
+        timeout: Joi.number().integer().optional().default(250).min(0),
+        retries: Joi.number().integer().optional().default(3).min(0),
       }).unknown(false).requiredKeys('url')
     }).unknown(false).default({}), // default api.services to an empty object
     defaultParameters: Joi.object().keys({

service/configurations/Interpolation.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class Language extends ServiceConfiguration {
+  constructor(o) {
+    super('interpolation', o);
+  }
+
+  getParameters(req, hit) {
+    return {
+      number: req.clean.parsed_text.number,
+      street: hit.address_parts.street || req.clean.parsed_text.street,
+      lat: hit.center_point.lat,
+      lon: hit.center_point.lon
+    };
+
+  }
+
+  getUrl(req) {
+    return url.resolve(this.baseUrl, 'search/geojson');
+  }
+
+}
+
+module.exports = Language;

service/configurations/Language.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class Language extends ServiceConfiguration {
+  constructor(o) {
+    super('language', o);
+  }
+
+  getParameters(req, res) {
+    // find all the values for all keys with names that end with '_id'
+    const ids = _.get(res, 'data', []).reduce((acc, doc) => {
+      Array.prototype.push.apply(acc, _.values(_.pickBy(doc.parent, (v, k) => _.endsWith(k, '_id') ) ) );
+      return acc;
+    }, []);
+
+    return {
+      // arrays will be nested, so flatten first, then uniqify, and finally join elements with comma
+      ids: _.uniq(_.flattenDeep(ids)).join(',')
+    };
+
+  }
+
+  getUrl(req) {
+    return url.resolve(this.baseUrl, 'parser/findbyid');
+  }
+
+}
+
+module.exports = Language;

service/configurations/PlaceHolder.js

@@ -12,9 +12,17 @@ class PlaceHolder extends ServiceConfiguration {
   }
 
   getParameters(req) {
-    const parameters = {
+    const parameters = {};
-      text: req.clean.text
+
-    };
+    if (_.has(req.clean.parsed_text, 'street')) {
+      // assemble all these fields into a space-delimited string
+      parameters.text = _.values(_.pick(req.clean.parsed_text,
+        ['neighbourhood', 'borough', 'city', 'county', 'state', 'country'])).join(' ');
+
+    } else {
+      parameters.text = req.clean.text;
+
+    }
 
     if (_.has(req.clean, 'lang.iso6393')) {
       parameters.lang = req.clean.lang.iso6393;

service/interpolation.js

@@ -1,118 +0,0 @@
-
-var logger = require( 'pelias-logger' ).get( 'api' ),
-    request = require( 'superagent' ),
-    peliasConfig = require( 'pelias-config' );
-
-/**
-
-  street address interpolation service client
-
-  this file provides several different 'transports' which can be used to access the interpolation
-  service, either directly from disk or via a network connnection.
-
-  the exported method for this module checks pelias-config for a configuration block such as:
-
-  "interpolation": {
-    "client": {
-      "adapter": "http",
-      "host": "http://localhost:4444"
-    }
-  }
-
-  for more info on running the service see: https://github.com/pelias/interpolation
-
-**/
-
-/**
-  NullTransport
-
-  disables the service completely
-**/
-function NullTransport(){}
-NullTransport.prototype.query = function( coord, number, street, cb ){
-  cb(); // no-op
-};
-
-/**
-  RequireTransport
-
-  allows the api to be used by simply requiring the module
-**/
-function RequireTransport( addressDbPath, streetDbPath ){
-  try {
-    var lib = require('pelias-interpolation'); // lazy load dependency
-    this.query = lib.api.search( addressDbPath, streetDbPath );
-  } catch( e ){
-    logger.error( 'RequireTransport: failed to connect to interpolation service' );
-  }
-}
-RequireTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'interpolation: transport not connected' );
-};
-
-/**
-  HttpTransport
-
-  allows the api to be used via a remote web service
-**/
-function HttpTransport( host, settings ){
-  this.query = function( coord, number, street, cb ){
-    request
-      .get( host + '/search/geojson' )
-      .set( 'Accept', 'application/json' )
-      .query({ lat: coord.lat, lon: coord.lon, number: number, street: street })
-      .timeout( settings && settings.timeout || 1000 )
-      .end( function( err, res ){
-        if( err || !res ){ return cb( err ); }
-        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
-        return cb( null, res.body );
-      });
-  };
-}
-HttpTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'interpolation: transport not connected' );
-};
-
-/**
-  Setup
-
-  allows instantiation of transport depending on configuration and preference
-**/
-module.exports.search = function setup(){
-
-  // user config
-  var config = peliasConfig.generate();
-
-  // ensure config variables set correctly
-  if( !config.hasOwnProperty('interpolation') || !config.interpolation.hasOwnProperty('client') ){
-    logger.warn( 'interpolation: configuration not found' );
-  }
-
-  // valid configuration found
-  else {
-
-    // get adapter settings from config
-    var settings = config.interpolation.client;
-
-    // http adapter
-    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
-      logger.info( 'interpolation: using http transport:', settings.host );
-      if( settings.hasOwnProperty('timeout') ){
-        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
-      }
-      return new HttpTransport( settings.host );
-    }
-
-    // require adapter
-    else if( 'require' === settings.adapter ){
-      if( settings.hasOwnProperty('streetdb') && settings.hasOwnProperty('addressdb') ){
-        logger.info( 'interpolation: using require transport' );
-        return new RequireTransport( settings.addressdb, settings.streetdb );
-      }
-    }
-  }
-
-  // default adapter
-  logger.info( 'interpolation: using null transport' );
-  return new NullTransport();
-};

service/language.js

@@ -1,93 +0,0 @@
-
-var logger = require( 'pelias-logger' ).get( 'api' ),
-    request = require( 'superagent' ),
-    peliasConfig = require( 'pelias-config' );
-
-/**
-
-  language subsitution service client
-
-  this file provides a 'transport' which can be used to access the language
-  service via a network connnection.
-
-  the exported method for this module checks pelias-config for a configuration block such as:
-
-  "language": {
-    "client": {
-      "adapter": "http",
-      "host": "http://localhost:6100"
-    }
-  }
-
-  for more info on running the service see: https://github.com/pelias/placeholder
-
-**/
-
-/**
-  NullTransport
-
-  disables the service completely
-**/
-function NullTransport(){}
-NullTransport.prototype.query = function( ids, cb ){
-  cb(); // no-op
-};
-
-/**
-  HttpTransport
-
-  allows the api to be used via a remote web service
-**/
-function HttpTransport( host, settings ){
-  this.query = function( ids, cb ){
-    request
-      .get( host + '/parser/findbyid' )
-      .set( 'Accept', 'application/json' )
-      .query({ ids: Array.isArray( ids ) ? ids.join(',') : '' })
-      .timeout( settings && settings.timeout || 1000 )
-      .end( function( err, res ){
-        if( err || !res ){ return cb( err ); }
-        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
-        return cb( null, res.body );
-      });
-  };
-}
-HttpTransport.prototype.query = function( coord, number, street, cb ){
-  throw new Error( 'language: transport not connected' );
-};
-
-/**
-  Setup
-
-  allows instantiation of transport depending on configuration and preference
-**/
-module.exports.findById = function setup(){
-
-  // user config
-  var config = peliasConfig.generate();
-
-  // ensure config variables set correctly
-  if( !config.hasOwnProperty('language') || !config.language.hasOwnProperty('client') ){
-    logger.warn( 'language: configuration not found' );
-  }
-
-  // valid configuration found
-  else {
-
-    // get adapter settings from config
-    var settings = config.language.client;
-
-    // http adapter
-    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
-      logger.info( 'language: using http transport:', settings.host );
-      if( settings.hasOwnProperty('timeout') ){
-        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
-      }
-      return new HttpTransport( settings.host );
-    }
-  }
-
-  // default adapter
-  logger.info( 'language: using null transport' );
-  return new NullTransport();
-};

test/unit/controller/libpostal.js

@@ -0,0 +1,290 @@
+'use strict';
+
+const proxyquire =  require('proxyquire').noCallThru();
+const _ = require('lodash');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', t => {
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => undefined
+      }
+    });
+
+    t.equal(typeof controller, 'function', 'libpostal is a function');
+    t.equal(typeof controller(), 'function', 'libpostal returns a controller');
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.should_execute = (test, common) => {
+  test('should_execute returning false should not call text-analyzer', t => {
+    const should_execute = (req, res) => {
+      // req and res should be passed to should_execute
+      t.deepEquals(req, {
+        clean: {
+          text: 'original query'
+        }
+      });
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
+
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => {
+          t.fail('parse should not have been called');
+        }
+      }
+    })(should_execute);
+
+    const req = {
+      clean: {
+        text: 'original query'
+      }
+    };
+    const res = { b: 2 };
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          text: 'original query'
+        }
+      }, 'req should not have been modified');
+      t.deepEquals(res, { b: 2 });
+      t.end();
+    });
+
+  });
+
+  test('should_execute returning false should not call text-analyzer', t => {
+    t.plan(5);
+
+    const should_execute = (req, res) => {
+      // req and res should be passed to should_execute
+      t.deepEquals(req, {
+        clean: {
+          text: 'original query'
+        }
+      });
+      t.deepEquals(res, { b: 2 });
+      return true;
+    };
+
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: (query) => {
+          t.equals(query, 'original query');
+          return undefined;
+        }
+      }
+    })(should_execute);
+
+    const req = {
+      clean: {
+        text: 'original query'
+      }
+    };
+    const res = { b: 2 };
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          text: 'original query'
+        }
+      }, 'req should not have been modified');
+      t.deepEquals(res, { b: 2 });
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.tests.parse_is_called = (test, common) => {
+  test('parse returning undefined should not overwrite clean.parsed_text', t => {
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => undefined
+      }
+    })(() => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'original parsed_text'
+      }
+    };
+    const res = 'this is the response';
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          parsed_text: 'original parsed_text'
+        }
+      });
+      t.deepEquals(res, 'this is the response');
+      t.end();
+    });
+
+  });
+
+  test('parse returning something should overwrite clean.parsed_text', t => {
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => 'replacement parsed_text'
+      }
+    })(() => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'original parsed_text'
+      }
+    };
+    const res = 'this is the response';
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          parsed_text: 'replacement parsed_text'
+        }
+      });
+      t.deepEquals(res, 'this is the response');
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.tests.iso2_conversion = (test, common) => {
+  test('no country in parse response should not leave country unset', t => {
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => ({
+          locality: 'this is the locality'
+        })
+      },
+      'iso3166-1': {
+        is2: () => t.fail('should not have been called'),
+        to3: () => t.fail('should not have been called')
+      }
+    })(() => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'original parsed_text'
+      }
+    };
+    const res = 'this is the response';
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          parsed_text: {
+            locality: 'this is the locality'
+          }
+        }
+      });
+      t.deepEquals(res, 'this is the response');
+      t.end();
+    });
+
+  });
+
+  test('unknown country should not be converted', t => {
+    t.plan(3);
+
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => ({
+          country: 'unknown country code'
+        })
+      },
+      'iso3166-1': {
+        is2: country => {
+          t.equals(country, 'UNKNOWN COUNTRY CODE');
+          return false;
+        },
+        to3: () => t.fail('should not have been called')
+      }
+    })(() => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'original parsed_text'
+      }
+    };
+    const res = 'this is the response';
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          parsed_text: {
+            country: 'unknown country code'
+          }
+        }
+      });
+      t.deepEquals(res, 'this is the response');
+      t.end();
+    });
+
+  });
+
+  test('ISO2 country should be converted to ISO3', t => {
+    t.plan(4);
+
+    const controller = proxyquire('../../../controller/libpostal', {
+      'pelias-text-analyzer': {
+        parse: () => ({
+          country: 'ISO2 COUNTRY CODE'
+        })
+      },
+      'iso3166-1': {
+        is2: country => {
+          t.equals(country, 'ISO2 COUNTRY CODE');
+          return true;
+        },
+        to3: country => {
+          t.equals(country, 'ISO2 COUNTRY CODE');
+          return 'ISO3 COUNTRY CODE';
+        }
+      }
+    })(() => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'original parsed_text'
+      }
+    };
+    const res = 'this is the response';
+
+    controller(req, res, () => {
+      t.deepEquals(req, {
+        clean: {
+          parsed_text: {
+            country: 'ISO3 COUNTRY CODE'
+          }
+        }
+      });
+      t.deepEquals(res, 'this is the response');
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+
+  function test(name, testFunction) {
+    return tape(`GET /libpostal ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/placeholder.js

@@ -30,7 +30,7 @@ module.exports.tests.should_execute = (test, common) => {
       return false;
     };
 
-    const controller = placeholder(placeholder_service, should_execute);
+    const controller = placeholder(placeholder_service, true, should_execute);
 
     const req = { a: 1 };
     const res = { b: 2 };
@@ -52,7 +52,7 @@ module.exports.tests.should_execute = (test, common) => {
       callback(null, []);
     };
 
-    const controller = placeholder(placeholder_service, _.constant(true));
+    const controller = placeholder(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { b: 2 };
@@ -206,7 +206,7 @@ module.exports.tests.success = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -324,7 +324,7 @@ module.exports.tests.success = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -387,7 +387,7 @@ module.exports.tests.success = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -447,7 +447,7 @@ module.exports.tests.success = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -512,7 +512,7 @@ module.exports.tests.success = (test, common) => {
 
       const controller = proxyquire('../../../controller/placeholder', {
         'pelias-logger': logger
-      })(placeholder_service, _.constant(true));
+      })(placeholder_service, true, () => true);
 
       const req = { param1: 'param1 value' };
       const res = { };
@@ -577,7 +577,7 @@ module.exports.tests.success = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -742,7 +742,7 @@ module.exports.tests.result_filtering = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       param1: 'param1 value',
@@ -804,6 +804,141 @@ module.exports.tests.result_filtering = (test, common) => {
 
   });
 
+  test('when geometric_filters_apply is false, boundary.rect should not apply', (t) => {
+    const logger = require('pelias-mock-logger')();
+
+    const placeholder_service = (req, callback) => {
+      t.deepEqual(req, {
+        param1: 'param1 value',
+        clean: {
+          'boundary.rect.min_lat': -1,
+          'boundary.rect.max_lat': 1,
+          'boundary.rect.min_lon': -1,
+          'boundary.rect.max_lon': 1
+        }
+      });
+
+      const response = [
+        {
+          // inside bbox
+          id: 1,
+          name: 'name 1',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: 0,
+            lon: 0
+          }
+        },
+        {
+          // outside bbox
+          id: 2,
+          name: 'name 2',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: -2,
+            lon: 2
+          }
+        },
+        {
+          // outside bbox
+          id: 3,
+          name: 'name 3',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: 2,
+            lon: -2
+          }
+        }
+      ];
+
+      callback(null, response);
+    };
+
+    const should_execute = (req, res) => {
+      return true;
+    };
+
+    const controller = proxyquire('../../../controller/placeholder', {
+      'pelias-logger': logger
+    })(placeholder_service, false, () => true);
+
+    const req = {
+      param1: 'param1 value',
+      clean: {
+        'boundary.rect.min_lat': -1,
+        'boundary.rect.max_lat': 1,
+        'boundary.rect.min_lon': -1,
+        'boundary.rect.max_lon': 1
+      }
+    };
+    const res = { };
+
+    controller(req, res, () => {
+      const expected_res = {
+        meta: {
+          query_type: 'fallback'
+        },
+        data: [
+          {
+            _id: '1',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '1',
+            center_point: {
+              lat: 0,
+              lon: 0
+            },
+            name: {
+              'default': 'name 1'
+            },
+            phrase: {
+              'default': 'name 1'
+            }
+          },
+          {
+            _id: '2',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '2',
+            center_point: {
+              lat: -2,
+              lon: 2
+            },
+            name: {
+              'default': 'name 2'
+            },
+            phrase: {
+              'default': 'name 2'
+            }
+          },
+          {
+            _id: '3',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '3',
+            center_point: {
+              lat: 2,
+              lon: -2
+            },
+            name: {
+              'default': 'name 3'
+            },
+            phrase: {
+              'default': 'name 3'
+            }
+          }
+        ]
+      };
+
+      t.deepEquals(res, expected_res);
+      t.end();
+    });
+
+  });
+
   test('when boundary.circle is available, results outside of it should be removed', (t) => {
     const logger = require('pelias-mock-logger')();
 
@@ -927,7 +1062,7 @@ module.exports.tests.result_filtering = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       param1: 'param1 value',
@@ -988,6 +1123,139 @@ module.exports.tests.result_filtering = (test, common) => {
 
   });
 
+  test('when geometric_filters_apply is false, boundary.circle should not apply', (t) => {
+    const logger = require('pelias-mock-logger')();
+
+    const placeholder_service = (req, callback) => {
+      t.deepEqual(req, {
+        param1: 'param1 value',
+        clean: {
+          'boundary.circle.lat': 0,
+          'boundary.circle.lon': 0,
+          'boundary.circle.radius': 500
+        }
+      });
+
+      const response = [
+        {
+          // inside circle
+          id: 1,
+          name: 'name 1',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: 1,
+            lon: 1
+          }
+        },
+        {
+          // outside circle on +lon
+          id: 2,
+          name: 'name 2',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: -45,
+            lon: 45
+          }
+        },
+        {
+          // outside bbox on +lat
+          id: 3,
+          name: 'name 3',
+          placetype: 'neighbourhood',
+          geom: {
+            lat: 45,
+            lon: -45
+          }
+        }
+      ];
+
+      callback(null, response);
+    };
+
+    const should_execute = (req, res) => {
+      return true;
+    };
+
+    const controller = proxyquire('../../../controller/placeholder', {
+      'pelias-logger': logger
+    })(placeholder_service, false, () => true);
+
+    const req = {
+      param1: 'param1 value',
+      clean: {
+        'boundary.circle.lat': 0,
+        'boundary.circle.lon': 0,
+        'boundary.circle.radius': 500
+      }
+    };
+    const res = { };
+
+    controller(req, res, () => {
+      const expected_res = {
+        meta: {
+          query_type: 'fallback'
+        },
+        data: [
+          {
+            _id: '1',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '1',
+            center_point: {
+              lat: 1,
+              lon: 1
+            },
+            name: {
+              'default': 'name 1'
+            },
+            phrase: {
+              'default': 'name 1'
+            }
+          },
+          {
+            _id: '2',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '2',
+            center_point: {
+              lat: -45,
+              lon: 45
+            },
+            name: {
+              'default': 'name 2'
+            },
+            phrase: {
+              'default': 'name 2'
+            }
+          },
+          {
+            _id: '3',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '3',
+            center_point: {
+              lat: 45,
+              lon: -45
+            },
+            name: {
+              'default': 'name 3'
+            },
+            phrase: {
+              'default': 'name 3'
+            }
+          }
+        ]
+      };
+
+      t.deepEquals(res, expected_res);
+      t.end();
+    });
+
+  });
+
   test('only results matching explicit layers should be returned', (t) => {
     const logger = mock_logger();
 
@@ -1062,7 +1330,7 @@ module.exports.tests.result_filtering = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       param1: 'param1 value',
@@ -1143,14 +1411,17 @@ module.exports.tests.result_filtering = (test, common) => {
 
   });
 
-  test('only synthesized docs matching explicit boundary.country should be returned', (t) => {
+  test('if req.clean.parsed_text contains street, don\'t filter on anything', (t) => {
-    const logger = require('pelias-mock-logger')();
+    const logger = mock_logger();
 
     const placeholder_service = (req, callback) => {
       t.deepEqual(req, {
         param1: 'param1 value',
         clean: {
-          'boundary.country': 'ABC'
+          layers: ['neighbourhood'],
+          parsed_text: {
+            street: 'street value'
+          }
         }
       });
 
@@ -1158,15 +1429,147 @@ module.exports.tests.result_filtering = (test, common) => {
         {
           id: 1,
           name: 'name 1',
-          placetype: 'locality',
+          placetype: 'neighbourhood',
-          lineage: [
+          lineage: [ {} ],
-            {
+          geom: {
-              country: {
+            area: 1,
-                id: 1,
+            lat: 14.141414,
-                name: 'country name 1',
+            lon: 41.414141
-                abbr: 'ABC'
+          }
-              }
+        },
-            },
+        {
+          id: 2,
+          name: 'name 2',
+          placetype: 'borough',
+          lineage: [ {} ],
+          geom: {
+            area: 2,
+            lat: 15.151515,
+            lon: 51.515151
+          }
+        },
+        {
+          id: 3,
+          name: 'name 3',
+          placetype: 'locality',
+          lineage: [ {} ],
+          geom: {
+            area: 3,
+            lat: 16.161616,
+            lon: 61.616161
+          }
+        }
+      ];
+
+      callback(null, response);
+    };
+
+    const controller = proxyquire('../../../controller/placeholder', {
+      'pelias-logger': logger
+    })(placeholder_service, true, () => true);
+
+    const req = {
+      param1: 'param1 value',
+      clean: {
+        layers: ['neighbourhood'],
+        parsed_text: {
+          street: 'street value'
+        }
+      }
+    };
+    const res = { };
+
+    controller(req, res, () => {
+      const expected_res = {
+        meta: {
+          query_type: 'fallback'
+        },
+        data: [
+          {
+            _id: '1',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '1',
+            center_point: {
+              lat: 14.141414,
+              lon: 41.414141
+            },
+            name: {
+              'default': 'name 1'
+            },
+            phrase: {
+              'default': 'name 1'
+            }
+          },
+          {
+            _id: '2',
+            _type: 'borough',
+            layer: 'borough',
+            source: 'whosonfirst',
+            source_id: '2',
+            center_point: {
+              lat: 15.151515,
+              lon: 51.515151
+            },
+            name: {
+              'default': 'name 2'
+            },
+            phrase: {
+              'default': 'name 2'
+            }
+          },
+          {
+            _id: '3',
+            _type: 'locality',
+            layer: 'locality',
+            source: 'whosonfirst',
+            source_id: '3',
+            center_point: {
+              lat: 16.161616,
+              lon: 61.616161
+            },
+            name: {
+              'default': 'name 3'
+            },
+            phrase: {
+              'default': 'name 3'
+            }
+          }
+        ]
+      };
+
+      t.deepEquals(res, expected_res);
+      t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:3]'));
+      t.end();
+    });
+
+  });
+
+  test('only synthesized docs matching explicit boundary.country should be returned', (t) => {
+    const logger = require('pelias-mock-logger')();
+
+    const placeholder_service = (req, callback) => {
+      t.deepEqual(req, {
+        param1: 'param1 value',
+        clean: {
+          'boundary.country': 'ABC'
+        }
+      });
+
+      const response = [
+        {
+          id: 1,
+          name: 'name 1',
+          placetype: 'locality',
+          lineage: [
+            {
+              country: {
+                id: 1,
+                name: 'country name 1',
+                abbr: 'ABC'
+              }
+            },
             {
               country: {
                 id: 2,
@@ -1215,7 +1618,7 @@ module.exports.tests.result_filtering = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       param1: 'param1 value',
@@ -1285,6 +1688,178 @@ module.exports.tests.result_filtering = (test, common) => {
 
   });
 
+  test('when geometric_filters_apply is false, boundary.country should not apply', (t) => {
+    const logger = require('pelias-mock-logger')();
+
+    const placeholder_service = (req, callback) => {
+      t.deepEqual(req, {
+        param1: 'param1 value',
+        clean: {
+          'boundary.country': 'ABC'
+        }
+      });
+
+      const response = [
+        {
+          id: 1,
+          name: 'name 1',
+          placetype: 'locality',
+          lineage: [
+            {
+              country: {
+                id: 1,
+                name: 'country name 1',
+                abbr: 'ABC'
+              }
+            },
+            {
+              country: {
+                id: 2,
+                name: 'country name 2',
+                abbr: 'DEF'
+              }
+            }
+          ],
+          geom: {
+            lat: 14.141414,
+            lon: 41.414141
+          }
+        },
+        {
+          id: 3,
+          name: 'name 3',
+          placetype: 'locality',
+          lineage: [
+            {
+              country: {
+                id: 3,
+                name: 'country name 3',
+                abbr: 'ABC'
+              }
+            }
+          ],
+          geom: {
+            lat: 15.151515,
+            lon: 51.515151
+          }
+        },
+        {
+          id: 4,
+          name: 'name 4',
+          placetype: 'locality',
+          lineage: [
+            {
+              country: {
+                id: 4,
+                name: 'country name 4',
+                abbr: 'GHI'
+              }
+            }
+          ],
+          geom: {
+            lat: 16.161616,
+            lon: 61.616161
+          }
+        }
+      ];
+
+      callback(null, response);
+    };
+
+    const controller = proxyquire('../../../controller/placeholder', {
+      'pelias-logger': logger
+    })(placeholder_service, false, () => true);
+
+    const req = {
+      param1: 'param1 value',
+      clean: {
+        'boundary.country': 'ABC'
+      }
+    };
+    const res = { };
+
+    controller(req, res, () => {
+      const expected_res = {
+        meta: {
+          query_type: 'fallback'
+        },
+        data: [
+          {
+            _id: '1',
+            _type: 'locality',
+            layer: 'locality',
+            source: 'whosonfirst',
+            source_id: '1',
+            center_point: {
+              lat: 14.141414,
+              lon: 41.414141
+            },
+            name: {
+              'default': 'name 1'
+            },
+            phrase: {
+              'default': 'name 1'
+            },
+            parent: {
+              country: ['country name 1', 'country name 2'],
+              country_id: ['1', '2'],
+              country_a: ['ABC', 'DEF']
+            }
+          },
+          {
+            _id: '3',
+            _type: 'locality',
+            layer: 'locality',
+            source: 'whosonfirst',
+            source_id: '3',
+            center_point: {
+              lat: 15.151515,
+              lon: 51.515151
+            },
+            name: {
+              'default': 'name 3'
+            },
+            phrase: {
+              'default': 'name 3'
+            },
+            parent: {
+              country: ['country name 3'],
+              country_id: ['3'],
+              country_a: ['ABC']
+            }
+          },
+          {
+            _id: '4',
+            _type: 'locality',
+            layer: 'locality',
+            source: 'whosonfirst',
+            source_id: '4',
+            center_point: {
+              lat: 16.161616,
+              lon: 61.616161
+            },
+            name: {
+              'default': 'name 4'
+            },
+            phrase: {
+              'default': 'name 4'
+            },
+            parent: {
+              country: ['country name 4'],
+              country_id: ['4'],
+              country_a: ['GHI']
+            }
+          }
+        ]
+      };
+
+      t.deepEquals(res, expected_res);
+      t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:3]'));
+      t.end();
+    });
+
+  });
+
 };
 
 module.exports.tests.lineage_errors = (test, common) => {
@@ -1325,7 +1900,7 @@ module.exports.tests.lineage_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1399,7 +1974,7 @@ module.exports.tests.lineage_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1472,7 +2047,7 @@ module.exports.tests.lineage_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1532,7 +2107,7 @@ module.exports.tests.geometry_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1591,7 +2166,7 @@ module.exports.tests.centroid_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1651,7 +2226,7 @@ module.exports.tests.centroid_errors = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = { param1: 'param1 value' };
     const res = { };
@@ -1721,7 +2296,7 @@ module.exports.tests.boundingbox_errors = (test, common) => {
 
       const controller = proxyquire('../../../controller/placeholder', {
         'pelias-logger': logger
-      })(placeholder_service, _.constant(true));
+      })(placeholder_service, true, () => true);
 
       const req = { param1: 'param1 value' };
       const res = { };
@@ -1773,7 +2348,7 @@ module.exports.tests.error_conditions = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       errors: []
@@ -1802,7 +2377,7 @@ module.exports.tests.error_conditions = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       errors: []
@@ -1827,7 +2402,7 @@ module.exports.tests.error_conditions = (test, common) => {
 
     const controller = proxyquire('../../../controller/placeholder', {
       'pelias-logger': logger
-    })(placeholder_service, _.constant(true));
+    })(placeholder_service, true, () => true);
 
     const req = {
       errors: []

test/unit/controller/predicates/has_parsed_text_properties.js

@@ -0,0 +1,163 @@
+'use strict';
+
+const _ = require('lodash');
+const has_parsed_text_properties = require('../../../../controller/predicates/has_parsed_text_properties');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.ok(_.isFunction(has_parsed_text_properties.all), 'has_parsed_text_properties.all is a function');
+    t.ok(_.isFunction(has_parsed_text_properties.any), 'has_parsed_text_properties.any is a function');
+    t.end();
+  });
+
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('all: defined request.clean.parsed_text.property should return true', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property: 'value'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: clean.parsed_text with any property should return true ', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property1: 'value1',
+          property2: 'value2'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.all('property2', 'property1')(req));
+    t.end();
+
+  });
+
+  test('any: defined request.clean.parsed_text.property should return true', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property: 'value'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: clean.parsed_text with any property should return true ', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property2: 'value2',
+          property3: 'value3'
+        }
+      }
+    };
+
+    t.ok(has_parsed_text_properties.any('property1', 'property3')(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('all: undefined request should return false', (t) => {
+    t.notOk(has_parsed_text_properties.all('property')());
+    t.end();
+
+  });
+
+  test('all: undefined request.clean should return false', (t) => {
+    const req = {};
+
+    t.notOk(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: undefined request.clean.parsed_text should return false', (t) => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(has_parsed_text_properties.all('property')(req));
+    t.end();
+
+  });
+
+  test('all: request.clean.parsed_text with none of the supplied properties should return false', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {
+          property1: 'value1'
+        }
+      }
+    };
+
+    t.notOk(has_parsed_text_properties.all('property1', 'property2')(req));
+    t.end();
+
+  });
+
+  test('any: undefined request should return false', (t) => {
+    t.notOk(has_parsed_text_properties.any('property')());
+    t.end();
+
+  });
+
+  test('any: undefined request.clean should return false', (t) => {
+    const req = {};
+
+    t.notOk(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: undefined request.clean.parsed_text should return false', (t) => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(has_parsed_text_properties.any('property')(req));
+    t.end();
+
+  });
+
+  test('any: request.clean.parsed_text with none of the supplied properties should return false', (t) => {
+    const req = {
+      clean: {
+        parsed_text: {}
+      }
+    };
+
+    t.notOk(has_parsed_text_properties.any('property1', 'property2')(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /has_parsed_text_properties ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/has_request_parameter.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const _ = require('lodash');
+const has_request_parameter = require('../../../../controller/predicates/has_request_parameter');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', t => {
+    t.equal(typeof has_request_parameter, 'function', 'has_request_parameter is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('request with specified parameter should return true', t => {
+    [[], {}, 'string value', 17].forEach(val => {
+      const req = {
+        clean: {
+          'parameter name': val
+        }
+      };
+
+      t.ok(has_request_parameter('parameter name')(req));
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('request with undefined clean should return false', t => {
+    const req = {};
+
+    t.notOk(has_request_parameter('parameter name')(req));
+    t.end();
+
+  });
+
+  test('request.clean without specified parameter should return false', t => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(has_request_parameter('parameter name')(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /has_request_parameter ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_addressit_parse.js

@@ -0,0 +1,73 @@
+'use strict';
+
+const _ = require('lodash');
+const is_addressit_parse = require('../../../../controller/predicates/is_addressit_parse');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', t => {
+    t.ok(_.isFunction(is_addressit_parse), 'is_addressit_parse is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('request.clean.parser=addressit should return true', t => {
+    const req = {
+      clean: {
+        parser: 'addressit'
+      }
+    };
+
+    t.ok(is_addressit_parse(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('undefined request should return false', t => {
+    t.notOk(is_addressit_parse(undefined));
+    t.end();
+  });
+
+  test('undefined request.clean should return false', t => {
+    const req = {};
+
+    t.notOk(is_addressit_parse(req));
+    t.end();
+  });
+
+  test('undefined request.clean.parser should return false', t => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(is_addressit_parse(req));
+    t.end();
+  });
+
+  test('non-\'addressit\' request.clean.parser should return false', t => {
+    const req = {
+      clean: {
+        parser: 'not addressit'
+      }
+    };
+
+    t.notOk(is_addressit_parse(req));
+    t.end();
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /is_addressit_parse ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_admin_only_analysis.js

@@ -14,7 +14,7 @@ module.exports.tests.interface = (test, common) => {
 
 module.exports.tests.true_conditions = (test, common) => {
   test('parsed_text with admin-only properties should return true', (t) => {
-    ['neighbourhood', 'borough', 'city', 'county', 'state', 'postalcode', 'country'].forEach((property) => {
+    ['neighbourhood', 'borough', 'city', 'county', 'state', 'country'].forEach((property) => {
       const req = {
         clean: {
           parsed_text: {}
@@ -47,7 +47,7 @@ module.exports.tests.false_conditions = (test, common) => {
   });
 
   test('parsed_text with non-admin properties should return false', (t) => {
-    ['number', 'street', 'query', 'category'].forEach((property) => {
+    ['number', 'street', 'query', 'category', 'postalcode'].forEach((property) => {
       const req = {
         clean: {
           parsed_text: {}

test/unit/controller/predicates/is_only_non_admin_layers.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const _ = require('lodash');
+const is_only_non_admin_layers = require('../../../../controller/predicates/is_only_non_admin_layers');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', t => {
+    t.equal(typeof is_only_non_admin_layers, 'function', 'is_only_non_admin_layers is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('request with specified parameter should return true', t => {
+    [
+      ['venue', 'address', 'street'],
+      ['venue', 'address'],
+      ['venue', 'street'],
+      ['address', 'street'],
+      ['venue'],
+      ['address'],
+      ['street']
+    ].forEach(layers => {
+      const req = {
+        clean: {
+          layers: layers
+        }
+      };
+
+      t.ok(is_only_non_admin_layers(req));
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('request with undefined clean should return false', t => {
+    const req = {};
+
+    t.notOk(is_only_non_admin_layers(req));
+    t.end();
+
+  });
+
+  test('request.clean without layers parameter should return false', t => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(is_only_non_admin_layers(req));
+    t.end();
+
+  });
+
+  test('request with empty layers should return false', t => {
+    const req = {
+      clean: {
+        layers: []
+      }
+    };
+
+    t.notOk(is_only_non_admin_layers(req));
+    t.end();
+
+  });
+
+  test('request.clean.layers without venue, address, or street should return false', t => {
+    const req = {
+      clean: {
+        layers: ['locality']
+      }
+    };
+
+    t.notOk(is_only_non_admin_layers(req));
+    t.end();
+
+  });
+
+  test('request.clean.layers with other layers besides venue, address, or street should return false', t => {
+    ['venue', 'address', 'street'].forEach(non_admin_layer => {
+      const req = {
+        clean: {
+          layers: ['locality', non_admin_layer]
+        }
+      };
+
+      t.notOk(is_only_non_admin_layers(req));
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /is_only_non_admin_layers ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_request_sources_only_whosonfirst.js

@@ -0,0 +1,107 @@
+'use strict';
+
+const _ = require('lodash');
+const is_request_sources_only_whosonfirst = require('../../../../controller/predicates/is_request_sources_only_whosonfirst');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.ok(_.isFunction(is_request_sources_only_whosonfirst), 'is_request_sources_only_whosonfirst is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('sources only \'whosonfirst\' should return true', (t) => {
+    const req = {
+      clean: {
+        sources: [
+          'whosonfirst'
+        ]
+      }
+    };
+
+    t.ok(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('undefined req should return false', (t) => {
+    t.notOk(is_request_sources_only_whosonfirst(undefined));
+    t.end();
+
+  });
+
+  test('undefined req.clean should return false', (t) => {
+    const req = {};
+
+    t.notOk(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+  test('undefined req.clean.sources should return false', (t) => {
+    const req = {
+      clean: {}
+    };
+
+    t.notOk(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+  test('empty req.clean.sources should return false', (t) => {
+    const req = {
+      clean: {
+        sources: []
+      }
+    };
+
+    t.notOk(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+  test('sources not \'whosonfirst\' should return false', (t) => {
+    const req = {
+      clean: {
+        sources: [
+          'not whosonfirst'
+        ]
+      }
+    };
+
+    t.notOk(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+  test('sources other than \'whosonfirst\' should return false', (t) => {
+    const req = {
+      clean: {
+        sources: [
+          'whosonfirst', 'not whosonfirst'
+        ]
+      }
+    };
+
+    t.notOk(is_request_sources_only_whosonfirst(req));
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /is_request_sources_only_whosonfirst ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_service_enabled.js

@@ -1,42 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-const is_service_enabled = require('../../../../controller/predicates/is_service_enabled');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = (test, common) => {
-  test('valid interface', (t) => {
-    t.equal(typeof is_service_enabled, 'function', 'is_service_enabled  is a function');
-    t.equal(typeof is_service_enabled(), 'function', 'is_service_enabled() is a function');
-    t.end();
-  });
-};
-
-module.exports.tests.true_conditions = (test, common) => {
-  test('string uri should return true', (t) => {
-    t.ok(is_service_enabled('pip uri')());
-    t.end();
-
-  });
-
-};
-
-module.exports.tests.false_conditions = (test, common) => {
-  test('undefined uri should return false', (t) => {
-    t.notOk(is_service_enabled()());
-    t.end();
-
-  });
-
-};
-
-module.exports.all = (tape, common) => {
-  function test(name, testFunction) {
-    return tape(`GET /is_service_enabled ${name}`, testFunction);
-  }
-
-  for( const testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/controller/search_with_ids.js

@@ -0,0 +1,570 @@
+'use strict';
+
+const setup = require('../../../controller/search_with_ids');
+const proxyquire =  require('proxyquire').noCallThru();
+const mocklogger = require('pelias-mock-logger');
+const _ = require('lodash');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.ok(_.isFunction(setup), 'setup is a function');
+    t.ok(_.isFunction(setup()), 'setup returns a controller');
+    t.end();
+  });
+};
+
+module.exports.tests.success = (test, common) => {
+  test('successful request to search service should replace data and meta', (t) => {
+    t.plan(5);
+
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+      type: 'this is the query type'
+    });
+
+    // a controller that validates the esclient and cmd that was passed to the search service
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        t.equal(esclient, 'this is the esclient');
+        t.deepEqual(cmd, {
+          index: 'indexName value',
+          searchType: 'dfs_query_then_fetch',
+          body: 'this is the query body'
+        });
+
+        const docs = [
+          { name: 'replacement result #1'},
+          { name: 'replacement result #2'}
+        ];
+        const meta = { key: 'replacement meta value' };
+
+        callback(undefined, docs, meta);
+      },
+      'pelias-logger': logger
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {
+      data: [
+        { name: 'original result #1'},
+        { name: 'original result #2'}
+      ],
+      meta: {
+        key: 'original meta value'
+      }
+    };
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [],
+        warnings: []
+      });
+      t.deepEquals(res, {
+        data: [
+          { name: 'replacement result #1'},
+          { name: 'replacement result #2'}
+        ],
+        meta: {
+          key: 'replacement meta value',
+          query_type: 'this is the query type'
+        }
+      });
+
+      t.ok(logger.isInfoMessage('[controller:search] [queryType:this is the query type] [es_result_count:2]'));
+
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('undefined meta should set empty object into res', (t) => {
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+      type: 'this is the query type'
+    });
+
+    // a controller that validates the esclient and cmd that was passed to the search service
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        const docs = [
+          { name: 'replacement result #1'},
+          { name: 'replacement result #2'}
+        ];
+
+        callback(undefined, docs, undefined);
+      },
+      'pelias-logger': logger
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {
+      data: [
+        { name: 'original result #1'},
+        { name: 'original result #2'}
+      ],
+      meta: {
+        key: 'original meta value'
+      }
+    };
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [],
+        warnings: []
+      });
+      t.deepEquals(res, {
+        data: [
+          { name: 'replacement result #1'},
+          { name: 'replacement result #2'}
+        ],
+        meta: {
+          query_type: 'this is the query type'
+        }
+      });
+
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('undefined docs in response should not overwrite existing results', (t) => {
+    t.plan(1+3); // ensures that search service was called, then req+res+logger tests
+
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+      type: 'this is the query type'
+    });
+
+    // a controller that validates the esclient and cmd that was passed to the search service
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        t.pass('search service was called');
+
+        const meta = { key: 'new value' };
+
+        callback(undefined, undefined, meta);
+      },
+      'pelias-logger': logger
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {
+      data: [
+        { id: 1 },
+        { id: 2 }
+      ],
+      meta: {
+        key: 'value'
+      }
+    };
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [],
+        warnings: []
+      });
+      t.deepEquals(res, {
+        data: [
+          { id: 1 },
+          { id: 2 }
+        ],
+        meta: { key: 'value' }
+      });
+
+      t.notOk(logger.isInfoMessage(/[controller:search] [queryType:this is the query type] [es_result_count:0]/));
+
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('empty docs in response should not overwrite existing results', (t) => {
+    t.plan(4);
+
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+      type: 'this is the query type'
+    });
+
+    // a controller that validates the esclient and cmd that was passed to the search service
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        t.pass('search service was called');
+
+        const meta = { key: 'value' };
+
+        callback(undefined, [], meta);
+      }
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {
+      data: [
+        { name: 'pre-existing result #1' },
+        { name: 'pre-existing result #2' }
+      ],
+      meta: {
+        key: 'value'
+      }
+    };
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [],
+        warnings: []
+      });
+      t.deepEquals(res, {
+        data: [
+          { name: 'pre-existing result #1' },
+          { name: 'pre-existing result #2' }
+        ],
+        meta: { key: 'value' }
+      });
+
+      t.notOk(logger.isInfoMessage(/[controller:search] [queryType:this is the query type] [es_result_count:0]/));
+
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('successful request on retry to search service should log info message', (t) => {
+    t.plan(3+2+2); // 3 search service calls, 2 log messages, 1 req, 1 res
+
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+      type: 'this is the query type'
+    });
+
+    let searchServiceCallCount = 0;
+
+    const timeoutError = {
+      status: 408,
+      displayName: 'RequestTimeout',
+      message: 'Request Timeout after 17ms'
+    };
+
+    // a controller that validates the esclient and cmd that was passed to the search service
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        t.pass('search service was called');
+
+        if (searchServiceCallCount < 2) {
+          // note that the searchService got called
+          searchServiceCallCount++;
+          callback(timeoutError);
+        } else {
+          const docs = [
+            { name: 'replacement result #1'},
+            { name: 'replacement result #2'}
+          ];
+          const meta = { key: 'replacement meta value' };
+
+          callback(undefined, docs, meta);
+        }
+
+      },
+      'pelias-logger': logger
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {
+      data: [
+        { name: 'original result #1'},
+        { name: 'original result #2'}
+      ],
+      meta: {
+        key: 'original meta value'
+      }
+    };
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [],
+        warnings: []
+      });
+      t.deepEquals(res, {
+        data: [
+          { name: 'replacement result #1'},
+          { name: 'replacement result #2'}
+        ],
+        meta: {
+          key: 'replacement meta value',
+          query_type: 'this is the query type'
+        }
+      });
+
+      t.ok(logger.isInfoMessage('[controller:search] [queryType:this is the query type] [es_result_count:2]'));
+      t.ok(logger.isInfoMessage('succeeded on retry 2'));
+
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+};
+
+module.exports.tests.service_errors = (test, common) => {
+  test('default # of request timeout retries should be 3', (t) => {
+    // test for 1 initial search service, 3 retries, 1 log messages, 1 req, and 1 res
+    t.plan(1 + 3 + 1 + 2);
+
+    const logger = mocklogger();
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({
+      body: 'this is the query body',
+    });
+
+    const timeoutError = {
+      status: 408,
+      displayName: 'RequestTimeout',
+      message: 'Request Timeout after 17ms'
+    };
+
+    // a controller that validates that the search service was called
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        // note that the searchService got called
+        t.pass('search service was called');
+
+        callback(timeoutError);
+      },
+      'pelias-logger': logger
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {};
+
+    const next = () => {
+      t.deepEqual(logger.getInfoMessages(), [
+        '[req]',
+        'request timed out on attempt 1, retrying',
+        'request timed out on attempt 2, retrying',
+        'request timed out on attempt 3, retrying'
+      ]);
+
+      t.deepEqual(req, {
+        clean: {},
+        errors: [timeoutError.message],
+        warnings: []
+      });
+      t.deepEqual(res, {});
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('explicit apiConfig.requestRetries should retry that many times', (t) => {
+    t.plan(1 + 17); // test for initial search service call and 17 retries
+
+    const config = {
+      indexName: 'indexName value',
+      requestRetries: 17
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({ });
+
+    const timeoutError = {
+      status: 408,
+      displayName: 'RequestTimeout',
+      message: 'Request Timeout after 17ms'
+    };
+
+    // a controller that validates that the search service was called
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        // note that the searchService got called
+        t.pass('search service was called');
+
+        callback(timeoutError);
+      }
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {};
+
+    controller(req, res, () => t.end() );
+
+  });
+
+  test('only status code 408 should be considered a retryable request', (t) => {
+    t.plan(2);
+
+    const config = {
+      indexName: 'indexName value',
+      requestRetries: 17
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({ });
+
+    const nonTimeoutError = {
+      status: 500,
+      displayName: 'InternalServerError',
+      message: 'an internal server error occurred'
+    };
+
+    // a controller that validates that the search service was called
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        // note that the searchService got called
+        t.pass('search service was called');
+
+        callback(nonTimeoutError);
+      }
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {};
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: [nonTimeoutError.message],
+        warnings: []
+      });
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+  test('string error should not retry and be logged as-is', (t) => {
+    t.plan(2); // service call + error is in req.errors
+
+    const config = {
+      indexName: 'indexName value'
+    };
+    const esclient = 'this is the esclient';
+    const query = () => ({ });
+
+    // a controller that validates that the search service was called
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': (esclient, cmd, callback) => {
+        // note that the searchService got called
+        t.pass('search service was called');
+
+        callback('this is an error string');
+      }
+    })(config, esclient, query, () => true );
+
+    const req = { clean: { }, errors: [], warnings: [] };
+    const res = {};
+
+    const next = () => {
+      t.deepEqual(req, {
+        clean: {},
+        errors: ['this is an error string'],
+        warnings: []
+      });
+      t.end();
+    };
+
+    controller(req, res, next);
+
+  });
+
+};
+
+module.exports.tests.should_execute = (test, common) => {
+  test('should_execute returning false and empty req.errors should call next', (t) => {
+    const esclient = () => t.fail('esclient should not have been called');
+    const query = () => t.fail('query should not have been called');
+    const should_execute = () => false;
+    const controller = setup( {}, esclient, query, should_execute );
+
+    const req = { };
+    const res = { };
+
+    const next = () => {
+      t.deepEqual(res, { });
+      t.end();
+    };
+    controller(req, res, next);
+
+  });
+
+};
+
+module.exports.tests.undefined_query = (test, common) => {
+  test('query returning undefined should not call service', (t) => {
+    t.plan(0, 'test will fail if search service actually gets called');
+
+    // a function that returns undefined
+    const query = () => undefined;
+
+    const controller = proxyquire('../../../controller/search_with_ids', {
+      '../service/search': () => {
+        t.fail('search service should not have been called');
+      }
+    })(undefined, undefined, query, () => true );
+
+    const next = () => t.end();
+
+    controller({}, {}, next);
+
+  });
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /search ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/debug.js

@@ -0,0 +1,95 @@
+const Debug = require('../../../helper/debug');
+
+module.exports.tests = {};
+
+module.exports.tests.debug = function(test, common) {
+  test('initialize the debugger with a name', (t) => {
+    const debugLog = new Debug('debugger');
+    t.deepEquals(debugLog.name, 'debugger');
+    t.end();
+  });
+
+  test('don\'t push debug message if enableDebug is false', (t) => {
+    const debugLog = new Debug('debugger');
+    const req = {
+      clean: {
+        enableDebug: false
+      }
+    };
+    debugLog.push(req, 'This should not be pushed');
+    t.deepEquals(req.debug, undefined);
+    t.end();
+  });
+
+  test('don\'t start timer if enableDebug is false', (t) => {
+    const debugLog = new Debug('debugger');
+    const req = {
+      clean: {
+        enableDebug: false
+      }
+    };
+    debugLog.beginTimer(req, 'This should not be pushed');
+    t.deepEquals(req.debug, undefined);
+    t.end();
+  });
+
+  test('don\'t push debug message if req.clean is empty', (t) => {
+    const debugLog = new Debug('debugger');
+    const req = {
+      clean: {}
+    };
+    debugLog.push('This should not be pushed');
+    t.deepEquals(req.debug, undefined);
+    t.end();
+  });
+
+  test('Push messages if enableDebug is true', (t) => {
+    const debugLog = new Debug('debugger');
+    const req = {
+      clean: {
+        enableDebug: true
+      }
+    };
+    const expected_req = {
+      debug: [
+        {
+          debugger: 'This should be pushed'
+        },
+        {
+          debugger: 'Timer Began: Timer 1'
+        }
+      ]
+    };
+    debugLog.push(req, 'This should be pushed');
+    debugLog.beginTimer(req, 'Timer 1');
+    t.end();
+  });
+
+  test('Timer should return positive number of milliseconds', (t) => {
+    const debugLog = new Debug('debugger');
+    const req = {
+      clean: {
+        enableDebug: true
+      }
+    };
+    debugLog.beginTimer(req);
+    setTimeout(() => {
+      debugLog.stopTimer(req);
+      t.deepEquals(parseInt(req.debug[1].debugger.slice(15, -3)) > 0, true);
+      t.end();
+    }, 2);
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[helper] debug: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/middleware/changeLanguage.js

@@ -1,256 +1,235 @@
+'use strict';
 
-var fs = require('fs'),
+const setup = require('../../../middleware/changeLanguage');
-    tmp = require('tmp'),
-    setup = require('../../../middleware/changeLanguage');
 const proxyquire =  require('proxyquire').noCallThru();
-
+const _  = require('lodash');
-// load middleware using the default pelias config
-var load = function(){
-  // adapter is driven by config
-  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-  process.env.PELIAS_CONFIG = tmpfile;
-  var middleware = setup();
-  delete process.env.PELIAS_CONFIG;
-  return middleware;
-};
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.interface = (test, common) => {
-  test('valid interface', function(t) {
+  test('valid interface', t => {
-    var middleware = load();
+    t.equal(typeof setup, 'function', 'setup is a function');
-    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(typeof setup(), 'function', 'setup returns a controller');
-    t.equal(middleware.length, 3, 'middleware is a function');
     t.end();
   });
 };
 
-module.exports.tests.isLanguageChangeRequired = function(test, common) {
+module.exports.tests.early_exit_conditions = (test, common) => {
-  test('invalid query - null req/res', function(t) {
+  test('should_execute returning false should not call service', t => {
-    var middleware = load();
+    t.plan(2, 'should_execute will assert 2 things');
-    middleware(null, null, t.end);
-  });
-
-  test('invalid query - no results', function(t) {
-    var req = { language: { iso6393: 'spa' } };
-    var res = {};
 
-    var middleware = load();
+    const service = () => {
-    middleware(req, res, function(){
+      t.fail('service should not have been called');
-      t.deepEqual( req, { language: { iso6393: 'spa' } } );
+    };
-      t.deepEqual( res, {} );
-      t.end();
-    });
-  });
 
-  test('invalid query - empty results', function(t) {
+    const should_execute = (req, res) => {
-    var req = { language: { iso6393: 'spa' } };
+      t.deepEquals(req, { a: 1 });
-    var res = { data: [] };
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
 
-    var middleware = load();
+    const controller = setup(service, should_execute);
-    middleware(req, res, function(){
-      t.deepEqual( req, { language: { iso6393: 'spa' } } );
-      t.deepEqual( res, { data: [] } );
-      t.end();
-    });
-  });
 
-  test('invalid query - no target language', function(t) {
+    controller({ a: 1 }, { b: 2 }, () => { });
-    var req = {};
-    var res = { data: [] };
 
-    var middleware = load();
-    middleware(req, res, function(){
-      t.deepEqual( req, {} );
-      t.deepEqual( res, { data: [] } );
-      t.end();
-    });
   });
+
 };
 
-// check the service is called and response mapped correctly
+module.exports.tests.error_conditions = (test, common) => {
-module.exports.tests.miss = function(test, common) {
+  test('service error should log and call next', t => {
-  test('miss', function(t) {
+    // (2) req/res were passed to service
+    // (1) error was logged
+    // (1) res was not modified
+    t.plan(4);
+
+    const service = (req, res, callback) => {
+      t.deepEquals(req, { a: 1 } );
+      t.deepEquals(res, { b: 2 } );
+      callback('this is an error');
+    };
 
-    var req = { language: { iso6393: 'spa' } };
+    const logger = require('pelias-mock-logger')();
-    var res = { data: [
-      {
-        layer: 'locality',
-        name: { default: 'London' },
-        parent: {
-          locality_id: [ 101750367 ],
-          locality: [ 'London' ]
-        }
-      },
-      {
-        layer: 'example',
-        name: { default: 'London' },
-        parent: {
-          locality_id: [ 101735809 ],
-          locaity: [ 'London' ]
-        }
-      }
-    ]};
 
-    var middleware = load();
+    const controller = proxyquire('../../../middleware/changeLanguage', {
+      'pelias-logger': logger
+    })(service, () => true);
 
-    // mock out the transport
+    const req = { a: 1 };
-    middleware.transport.query = function mock( ids, cb ){
+    const res = { b: 2 };
-      t.deepEqual( ids, [ '101735809', '101750367' ] );
-      t.equal( typeof cb, 'function' );
-      cb( 'error' );
-    };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.ok(logger.isErrorMessage('this is an error'));
-        {
+      t.deepEquals(res, { b: 2 }, 'res should not have been modified');
-          layer: 'locality',
-          name: { default: 'London' },
-          parent: {
-            locality_id: [ 101750367 ],
-            locality: [ 'London' ]
-          }
-        },
-        {
-          layer: 'example',
-          name: { default: 'London' },
-          parent: {
-            locality_id: [ 101735809 ],
-            locaity: [ 'London' ]
-          }
-        }
-      ]});
-      t.end();
     });
+
   });
-};
 
-// check the service is called and response mapped correctly
+};
-module.exports.tests.hit = function(test, common) {
-  test('hit', function(t) {
 
-    var req = { language: { iso6393: 'spa' } };
+module.exports.tests.success_conditions = (test, common) => {
-    var res = { data: [
+  test('translations should be mapped in', t => {
-      {
+    // (2) req/res were passed to service
-        layer: 'locality',
+    // (1) error was logged
-        name: { default: 'London' },
+    // (1) res was not modified
-        parent: {
+    // t.plan(4);
-          locality_id: [ 101750367 ],
+
-          locality: [ 'London' ]
+    const service = (req, res, callback) => {
+      const response = {
+        '1': {
+          names: {
+            'requested language': [
+              'replacement name for layer1'
+            ],
+            // this should be ignored
+            'another language': [
+              'name in another language'
+            ]
           }
         },
-      {
+        '2': {
-        layer: 'example',
+          names: {
-        name: { default: 'London' },
+            'requested language': [
-        parent: {
+              'replacement name for layer2',
-          locality_id: [ 101735809 ],
+              // this should be ignored
-          locality: [ 'London' ]
+              'another replacement name for layer2'
+            ]
+          }
+        },
+        '3': {
+          names: {
+            'requested language': [
+              'replacement name 1 for layer3'
+            ]
           }
+        },
+        '4': {
+          names: {
+            'requested language': [
+              'replacement name 2 for layer3'
+            ]
           }
-    ]};
+        },
-
+        '10': {
-    var middleware = load();
+          // has names but not in the requested language
-
+          names: {
-    // mock out the transport
+            'another language': [
-    middleware.transport.query = function mock( ids, cb ){
+              'replacement name for layer4'
-      t.deepEqual( ids, [ '101735809', '101750367' ] );
+            ]
-      t.equal( typeof cb, 'function' );
-      cb( null, {
-        '101750367': {
-          'names': {
-            'default':['London'],
-            'chi':['倫敦'],
-            'spa':['Londres'],
-            'eng':['London'],
-            'hin':['लंदन'],
-            'ara':['لندن'],
-            'por':['Londres'],
-            'ben':['লন্ডন'],
-            'rus':['Лондон'],
-            'jpn':['ロンドン'],
-            'kor':['런던']
           }
         },
-        '101735809': {
+        '11': {
-          'names':{
+          // no names
-            'default':['London'],
+        }
-            'eng':['London']
+      };
+
+      callback(null, response);
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/changeLanguage', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        lang: {
+          iso6393: 'requested language'
         }
       }
-      });
     };
 
-    middleware(req, res, function(){
+    const res = {
-      t.deepEqual( res, { data: [
+      data: [
+        // doc with 2 layer names that will be changed
         {
-          layer: 'locality',
+          name: {
-          name: { default: 'Londres' },
+            default: 'original name for 1st result'
+          },
+          layer: 'layer1',
           parent: {
-            locality_id: [ 101750367 ],
+            layer1_id: ['1'],
-            locality: [ 'Londres' ]
+            layer1: ['original name for layer1'],
+            layer2_id: ['2'],
+            layer2: ['original name for layer2']
           }
         },
+        // not sure how this would sneak in but check anyway
+        undefined,
+        // doc w/o parent
+        {},
+        // doc with only 1 layer name that will be changed and no default name change
         {
-          layer: 'example',
+          name: {
-          name: { default: 'London' },
+            default: 'original name for 2nd result'
+          },
+          layer: 'layer10',
           parent: {
-            locality_id: [ 101735809 ],
+            layer3_id: ['3', '4'],
-            locality: [ 'London' ]
+            layer3: ['original name 1 for layer3', 'original name 2 for layer3'],
+            // requested language not found for this id
+            layer10_id: ['10'],
+            layer10: ['original name for layer10'],
+            // no names for this id
+            layer11_id: ['11'],
+            layer11: ['original name for layer11'],
+            // no translations for this id
+            layer12_id: ['12'],
+            layer12: ['original name for layer12'],
+            // undefined id, will be skipped
+            layer13_id: [undefined],
+            layer13: ['original name for layer13']
           }
         }
-      ]});
+      ]
-      t.end();
+    };
-    });
-  });
 
-  test('empty array name translation should not change the value', t => {
+    controller(req, res, () => {
-    t.plan(2);
+      t.ok(logger.isDebugMessage('[language] [debug] missing translation requested language 10'));
+      t.ok(logger.isDebugMessage('[language] [debug] missing translation requested language 11'));
+      t.ok(logger.isDebugMessage('[language] [debug] failed to find translations for 12'));
 
-    const req = { language: { iso6393: 'ISO3 value' } };
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
-    const res = {
+
+      t.deepEquals(res, {
         data: [
           {
-          layer: 'locality',
+            name: {
-          name: { default: 'original name' },
+              default: 'replacement name for layer1'
+            },
+            layer: 'layer1',
             parent: {
-            locality_id: [ 123 ],
+              layer1_id: ['1'],
-            locality: [ 'original name' ]
+              layer1: ['replacement name for layer1'],
-          }
+              layer2_id: ['2'],
-        }
+              layer2: ['replacement name for layer2']
-      ]
-    };
-
-    const changeLanguage = proxyquire('../../../middleware/changeLanguage', {
-      '../service/language': {
-        findById: () => ({
-          query: (ids, callback) => {
-            t.deepEquals(ids, ['123']);
-            callback(null, {
-              '123': {
-                'names': {
-                  'ISO3 value':[]
-                }
-              }
-            });
-          }
-        })
             }
-    })();
+          },
-
+          undefined,
-    changeLanguage(req, res, () => {
+          {},
-      t.deepEqual( res, { data: [
           {
-          layer: 'locality',
             name: {
-            default: 'original name'
+              default: 'original name for 2nd result'
             },
+            layer: 'layer10',
             parent: {
-            locality_id: [ 123 ],
+              layer3_id: ['3', '4'],
-            locality: [ 'original name' ]
+              layer3: ['replacement name 1 for layer3', 'replacement name 2 for layer3'],
+              layer10_id: ['10'],
+              layer10: ['original name for layer10'],
+              layer11_id: ['11'],
+              layer11: ['original name for layer11'],
+              layer12_id: ['12'],
+              layer12: ['original name for layer12'],
+              layer13_id: [undefined],
+              layer13: ['original name for layer13']
             }
           }
-      ]});
+        ]
+      });
+
+      t.end();
 
     });
 
@@ -258,13 +237,13 @@ module.exports.tests.hit = function(test, common) {
 
 };
 
-module.exports.all = function (tape, common) {
+module.exports.all = (tape, common) => {
 
   function test(name, testFunction) {
-    return tape('[middleware] changeLanguage: ' + name, testFunction);
+    return tape(`GET /changeLanguage ${name}`, testFunction);
   }
 
-  for( var testCase in module.exports.tests ){
+  for( const testCase in module.exports.tests ){
     module.exports.tests[testCase](test, common);
   }
 };

test/unit/middleware/interpolate.js

@@ -1,275 +1,687 @@
+'use strict';
 
-var fs = require('fs'),
+const setup = require('../../../middleware/interpolate');
-    tmp = require('tmp'),
+const proxyquire =  require('proxyquire').noCallThru();
-    setup = require('../../../middleware/interpolate');
+const _  = require('lodash');
-
-// load middleware using the default pelias config
-var load = function(){
-  // adapter is driven by config
-  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
-  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
-  process.env.PELIAS_CONFIG = tmpfile;
-  var middleware = setup();
-  delete process.env.PELIAS_CONFIG;
-  return middleware;
-};
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.interface = (test, common) => {
-  test('valid interface', function(t) {
+  test('valid interface', t => {
-    var middleware = load();
+    t.equal(typeof setup, 'function', 'setup is a function');
-    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(typeof setup(), 'function', 'setup returns a controller');
-    t.equal(middleware.length, 3, 'middleware is a function');
     t.end();
   });
 };
 
-module.exports.tests.isAddressQuery = function(test, common) {
+module.exports.tests.early_exit_conditions = (test, common) => {
-  test('invalid address query - no parsed text', function(t) {
+  test('should_execute returning false should not call service', t => {
-    var req = { clean: {} };
+    t.plan(3, 'should_execute will assert 2 things + 1 for next() was called');
+
+    const service = () => {
+      t.fail('service should not have been called');
+    };
+
+    const should_execute = (req, res) => {
+      t.deepEquals(req, { a: 1 });
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
+
+    const controller = setup(service, should_execute);
+
+    controller({ a: 1 }, { b: 2 }, () => {
+      t.pass('next was called');
+    });
 
-    var middleware = load();
-    middleware(req, null, t.end);
   });
 
-  test('invalid address query - no number', function(t) {
-    var req = { clean: {
-      parsed_text: {
-        street: 'sesame st'
-      }}
 };
 
-    var middleware = load();
+module.exports.tests.error_conditions = (test, common) => {
-    middleware(req, null, t.end);
+  test('service error string should log and not modify any results', t => {
+    t.plan(2);
+
+    const service = (req, res, callback) => {
+      callback('this is an error', {
+        properties: {
+          number: 17,
+          source: 'OSM',
+          source_id: 'openstreetmap source id',
+          lat: 12.121212,
+          lon: 21.212121
+        }
       });
 
-  test('invalid address query - no street', function(t) {
-    var req = { clean: {
-      parsed_text: {
-        number: '1',
-      }}
     };
 
-    var middleware = load();
+    const logger = require('pelias-mock-logger')();
-    middleware(req, null, t.end);
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = { a: 1 };
+    const res = {
+      data: [
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {},
+          // bounding_box should be removed
+          bounding_box: {}
+        }
+      ]
+    };
+
+    controller(req, res, () => {
+      t.ok(logger.isErrorMessage('[middleware:interpolation] this is an error'));
+
+      t.deepEquals(res, {
+        data: [
+          {
+            id: 1,
+            layer: 'street',
+            name: {
+              default: 'street name 1'
+            },
+            address_parts: {},
+            // bounding_box should be removed
+            bounding_box: {}
+          }
+        ]
+      }, 'res should not have been modified');
+
     });
+
+  });
+
+  test('service error object should log message and not modify any results', t => {
+    t.plan(2);
+
+    const service = (req, res, callback) => {
+      callback({ message: 'this is an error' }, {
+        properties: {
+          number: 17,
+          source: 'OSM',
+          source_id: 'openstreetmap source id',
+          lat: 12.121212,
+          lon: 21.212121
+        }
+      });
+
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = { a: 1 };
+    const res = {
+      data: [
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {},
+          // bounding_box should be removed
+          bounding_box: {}
+        }
+      ]
     };
 
-// test results are correctly mapped to the transport
+    controller(req, res, () => {
-module.exports.tests.map = function(test, common) {
+      t.ok(logger.isErrorMessage('[middleware:interpolation] this is an error'));
-  test('documents mapped to transport: no hits', function(t) {
+
-    var req = { clean: {
+      t.deepEquals(res, {
-      parsed_text: {
+        data: [
-        number: '1',
+          {
-        street: 'sesame st'
+            id: 1,
-      }}
+            layer: 'street',
+            name: {
+              default: 'street name 1'
+            },
+            address_parts: {},
+            // bounding_box should be removed
+            bounding_box: {}
+          }
+        ]
+      }, 'res should not have been modified');
+
+    });
+
+  });
+
 };
-    var res = { data: [] };
 
-    var middleware = load();
+module.exports.tests.success_conditions = (test, common) => {
-    middleware(req, res, function(){
+  test('undefined res should not cause errors', t => {
-      t.deepEqual( res, { data: [] } );
+    const service = (req, res, callback) => {
+      t.fail('should not have been called');
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
+    };
+
+    controller(req, undefined, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
       t.end();
+
     });
+
   });
-  test('documents mapped to transport: no street layer hits', function(t) {
+
-    var req = { clean: {
+  test('undefined res.data should not cause errors', t => {
-      parsed_text: {
+    const service = (req, res, callback) => {
-        number: '1',
+      t.fail('should not have been called');
-        street: 'sesame st'
+    };
-      }}
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
     };
-    var res = { data: [{ layer: 'foo' }] };
 
-    var middleware = load();
+    const res = {};
-    middleware(req, res, function(){
+
-      t.deepEqual( res, { data: [{ layer: 'foo' }] } );
+    controller(req, res, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      t.deepEquals(res, {});
+
       t.end();
+
+    });
+
+  });
+
+  test('interpolated results should be mapped in', t => {
+    const service = (req, res, callback) => {
+      if (res.id === 1) {
+        callback(null, {
+          properties: {
+            number: 17,
+            source: 'Source Abbr 1',
+            source_id: 'source 1 source id',
+            lat: 12.121212,
+            lon: 21.212121
+          }
         });
+
+      } else if (res.id === 3) {
+        callback(null, {
+          properties: {
+            number: 18,
+            source: 'Source Abbr 2',
+            source_id: 'source 2 source id',
+            lat: 13.131313,
+            lon: 31.313131
+          }
+        });
+
+      } else if (res.id === 4) {
+        callback(null, {
+          properties: {
+            number: 19,
+            source: 'non-OSM/OA',
+            source_id: 'mixed source id',
+            lat: 14.141414,
+            lon: 41.414141
+          }
         });
+
+      } else {
+        t.fail(`unexpected id ${res.id}`);
+
+      }
+
     };
 
-// check the service is called and response mapped correctly
+    const logger = require('pelias-mock-logger')();
-module.exports.tests.miss = function(test, common) {
+
-  test('miss', function(t) {
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger,
+      '../helper/type_mapping': {
+        source_mapping: {
+          'source abbr 1': ['full source name 1'],
+          'source abbr 2': ['full source name 2']
+        }
+      }
+    })(service, () => true);
 
-    var req = { clean: {
+    const req = {
-      parsed_text: {
+      clean: {
-        number: '1',
+        parsed_text: 'this is req.clean.parsed_text'
-        street: 'sesame st'
+      }
-      }}
     };
-    var res = { data: [
+
+    const res = {
+      data: [
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {},
+          // will be replaced
+          source_id: 'original source_id',
+          // bounding_box should be removed
+          bounding_box: {}
+        },
+        {
+          id: 2,
+          layer: 'not street',
+          name: {
+            default: 'name 2'
+          },
+          address_parts: {}
+        },
         {
+          id: 3,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 3'
-        name: { default: 'example' }
+          },
+          address_parts: {}
+        },
+        {
+          id: 4,
+          layer: 'street',
+          name: {
+            default: 'street name 4'
+          },
+          address_parts: {}
         }
-    ]};
+      ]
+    };
 
-    var middleware = load();
+    controller(req, res, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/), 'timing should be info-logged');
 
-    // mock out the transport
+      // test debug messages very vaguely to avoid brittle tests
-    middleware.transport.query = function mock( coord, number, street, cb ){
+      t.ok(logger.isDebugMessage(/^\[interpolation\] \[hit\] this is req.clean.parsed_text \{.+?\}$/),
-      t.deepEqual( coord, res.data[0].center_point );
+        'hits should be debug-logged');
-      t.deepEqual( number, req.clean.parsed_text.number );
-      t.deepEqual( street, res.data[0].address_parts.street );
-      t.equal( typeof cb, 'function' );
-      cb( 'error' );
-    };
 
-    middleware(req, res, function(){
+      t.deepEquals(res, {
-      t.deepEqual( res, { data: [
+        data: [
           {
-          layer: 'street',
+            id: 1,
-          center_point: { lat: 1, lon: 1 },
+            layer: 'address',
-          address_parts: { street: 'sesame rd' },
+            match_type: 'interpolated',
-          name: { default: 'example' }
+            name: {
+              default: '17 street name 1'
+            },
+            source: 'full source name 1',
+            source_id: 'source 1 source id',
+            address_parts: {
+              number: 17
+            },
+            center_point: {
+              lat: 12.121212,
+              lon: 21.212121
             }
-      ]});
+          },
+          {
+            id: 3,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '18 street name 3'
+            },
+            source: 'full source name 2',
+            source_id: 'source 2 source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
+          },
+          {
+            id: 4,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '19 street name 4'
+            },
+            source: 'mixed',
+            source_id: 'mixed source id',
+            address_parts: {
+              number: 19
+            },
+            center_point: {
+              lat: 14.141414,
+              lon: 41.414141
+            }
+          },
+          {
+            id: 2,
+            layer: 'not street',
+            name: {
+              default: 'name 2'
+            },
+            address_parts: {}
+          }
+        ]
+      }, 'hits should be mapped in and res.data sorted with addresses first and non-addresses last');
+
       t.end();
+
+    });
+
   });
+
+  test('interpolation result without source_id should remove all together', t => {
+    const service = (req, res, callback) => {
+      if (res.id === 1) {
+        callback(null, {
+          properties: {
+            number: 17,
+            source: 'OA',
+            lat: 12.121212,
+            lon: 21.212121
+          }
         });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
+      }
+
     };
 
-// check the service is called and response mapped correctly
+    const logger = require('pelias-mock-logger')();
-module.exports.tests.hit = function(test, common) {
+
-  test('hit', function(t) {
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
 
-    var req = { clean: {
+    const req = {
-      parsed_text: {
+      clean: {
-        number: '1',
+        parsed_text: 'this is req.clean.parsed_text'
-        street: 'sesame st'
+      }
-      }}
     };
-    var res = { data: [
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
         {
+          id: 1,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 1'
-        name: { default: 'street name' },
+          },
-        source_id: '123456'
+          // will be removed
-      }
+          source_id: 'original source_id',
-    ]};
+          address_parts: {}
-
-    var middleware = load();
-
-    // mock out the transport
-    middleware.transport.query = function mock( coord, number, street, cb ){
-      t.deepEqual( coord, res.data[0].center_point );
-      t.deepEqual( number, req.clean.parsed_text.number );
-      t.deepEqual( street, res.data[0].address_parts.street );
-      t.equal( typeof cb, 'function' );
-      cb( null, {
-        properties: {
-          number: '100A',
-          source: 'OSM',
-          source_id: 'way:111111',
-          lat: 22.2,
-          lon: -33.3,
         }
-      });
+      ]
     };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      t.deepEquals(res, {
+        data: [
           {
+            id: 1,
             layer: 'address',
             match_type: 'interpolated',
-          center_point: { lat: 22.2, lon: -33.3 },
+            name: {
-          address_parts: { street: 'sesame rd', number: '100A' },
+              default: '17 street name 1'
-          name: { default: '100A street name' },
+            },
-          source: 'openstreetmap',
+            source: 'openaddresses',
-          source_id: 'way:111111'
+            address_parts: {
+              number: 17
+            },
+            center_point: {
+              lat: 12.121212,
+              lon: 21.212121
             }
-      ]});
+          }
+        ]
+      }, 'interpolation result did not have source_id so removed from source result');
+
       t.end();
+
     });
+
   });
+
+  test('undefined results should be skipped and not be fatal', t => {
+    const service = (req, res, callback) => {
+      if (res.id === 1) {
+        callback(null, undefined);
+
+      } else if (res.id === 2) {
+        callback(null, {
+          properties: {
+            number: 18,
+            source: 'OA',
+            source_id: 'openaddresses source id',
+            lat: 13.131313,
+            lon: 31.313131
+          }
+        });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
+      }
+
     };
 
-// check the service is called and response mapped correctly
+    const logger = require('pelias-mock-logger')();
-module.exports.tests.hit = function(test, common) {
+
-  test('hit', function(t) {
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
 
-    var req = { clean: {
+    const req = {
-      parsed_text: {
+      clean: {
-        number: '1',
+        parsed_text: 'this is req.clean.parsed_text'
-        street: 'sesame st'
+      }
-      }}
     };
-    var res = { data: [
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {}
+        },
         {
+          id: 2,
           layer: 'street',
-        center_point: { lat: 1, lon: 1 },
+          name: {
-        address_parts: { street: 'sesame rd' },
+            default: 'street name 2'
-        name: { default: 'street name' },
+          },
-        source_id: '123456'
+          address_parts: {}
+        }
+      ]
+    };
+
+    controller(req, res, () => {
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      // test debug messages very vaguely to avoid brittle tests
+      t.ok(logger.isDebugMessage('[interpolation] [miss] this is req.clean.parsed_text'));
+
+      t.deepEquals(res, {
+        data: [
+          {
+            id: 2,
+            layer: 'address',
+            match_type: 'interpolated',
+            name: {
+              default: '18 street name 2'
+            },
+            source: 'openaddresses',
+            source_id: 'openaddresses source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
           },
           {
+            id: 1,
             layer: 'street',
-        center_point: { lat: 2, lon: 2 },
+            name: {
-        address_parts: { street: 'sesame rd' },
+              default: 'street name 1'
-        name: { default: 'street name' },
+            },
-        source_id: '654321'
+            address_parts: {}
-      }
+          }
-    ]};
+        ]
-
+      }, 'only hits should have been mapped in');
-    var middleware = load();
+
-
+      t.end();
-    // mock out the transport
+
-    middleware.transport.query = function mock(coord, number, street, cb) {
+    });
-      if (coord.lat === 2) {
+
-        t.deepEqual(coord, res.data[1].center_point);
+  });
-        t.deepEqual(number, req.clean.parsed_text.number);
+
-        t.deepEqual(street, res.data[1].address_parts.street);
+  test('results missing \'properties\' should be skipped and not be fatal', t => {
-        t.equal(typeof cb, 'function');
+    const service = (req, res, callback) => {
-        return cb(null, {
+      if (res.id === 1) {
+        callback(null, {});
+
+      } else if (res.id === 2) {
+        callback(null, {
           properties: {
-            number: '100A',
+            number: 18,
-            source: 'OSM',
+            source: 'OA',
-            source_id: 'way:111111',
+            source_id: 'openaddresses source id',
-            lat: 22.2,
+            lat: 13.131313,
-            lon: -33.3,
+            lon: 31.313131
           }
         });
+
+      } else {
+        t.fail(`should not have been called with id ${res.id}`);
+
       }
-      else {
+
-        return cb('miss');
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../middleware/interpolate', {
+      'pelias-logger': logger
+    })(service, () => true);
+
+    const req = {
+      clean: {
+        parsed_text: 'this is req.clean.parsed_text'
+      }
+    };
+
+    const res = {
+      data: [
+        // doc with 2 layer names that will be changed
+        {
+          id: 1,
+          layer: 'street',
+          name: {
+            default: 'street name 1'
+          },
+          address_parts: {}
+        },
+        {
+          id: 2,
+          layer: 'street',
+          name: {
+            default: 'street name 2'
+          },
+          address_parts: {}
         }
+      ]
     };
 
-    middleware(req, res, function(){
+    controller(req, res, () => {
-      t.deepEqual( res, { data: [
+      t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages');
+      t.ok(logger.isInfoMessage(/\[interpolation\] \[took\] \d+ ms/));
+
+      // test debug messages very vaguely to avoid brittle tests
+      t.ok(logger.isDebugMessage('[interpolation] [miss] this is req.clean.parsed_text'));
+
+      t.deepEquals(res, {
+        data: [
           {
+            id: 2,
             layer: 'address',
             match_type: 'interpolated',
-          center_point: { lat: 22.2, lon: -33.3 },
+            name: {
-          address_parts: { street: 'sesame rd', number: '100A' },
+              default: '18 street name 2'
-          name: { default: '100A street name' },
+            },
-          source: 'openstreetmap',
+            source: 'openaddresses',
-          source_id: 'way:111111'
+            source_id: 'openaddresses source id',
+            address_parts: {
+              number: 18
+            },
+            center_point: {
+              lat: 13.131313,
+              lon: 31.313131
+            }
           },
           {
+            id: 1,
             layer: 'street',
-          center_point: { lat: 1, lon: 1 },
+            name: {
-          address_parts: { street: 'sesame rd' },
+              default: 'street name 1'
-          name: { default: 'street name' },
+            },
-          source_id: '123456'
+            address_parts: {}
           }
-      ]});
+        ]
-      t.end();
       });
+
+      t.end();
+
+    }, 'only hits should have been mapped in');
+
   });
-};
 
+};
 
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('[middleware] interpolate: ' + name, testFunction);
+    return tape(`[middleware] interpolate: ${name}`, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/query/MockQuery.js

@@ -0,0 +1,27 @@
+'use strict';
+
+module.exports = class MockQuery {
+  constructor() {
+    this._score_functions = [];
+    this._filter_functions = [];
+  }
+
+  render(vs) {
+    return {
+      vs: vs,
+      score_functions: this._score_functions,
+      filter_functions: this._filter_functions
+    };
+  }
+
+  score(view) {
+    this._score_functions.push(view);
+    return this;
+  }
+
+  filter(view) {
+    this._filter_functions.push(view);
+    return this;
+  }
+
+};

test/unit/query/address_search_using_ids.js

@@ -0,0 +1,553 @@
+const generateQuery = require('../../../query/address_search_using_ids');
+const _ = require('lodash');
+const proxyquire = require('proxyquire').noCallThru();
+const mock_logger = require('pelias-mock-logger');
+const MockQuery = require('./MockQuery');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.ok(_.isFunction(generateQuery));
+    t.end();
+  });
+};
+
+// helper for canned views
+const views = {
+  focus_only_function: () => 'focus_only_function',
+  boundary_country: 'boundary_country view',
+  boundary_circle: 'boundary_circle view',
+  boundary_rect: 'boundary_rect view',
+  sources: 'sources view'
+};
+
+module.exports.tests.base_query = (test, common) => {
+  test('basic', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      }
+    };
+    const res = {
+      data: []
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.type, 'fallback');
+
+    t.equals(generatedQuery.body.vs.var('input:housenumber').toString(), 'housenumber value');
+    t.equals(generatedQuery.body.vs.var('input:street').toString(), 'street value');
+    t.notOk(generatedQuery.body.vs.isset('sources'));
+    t.equals(generatedQuery.body.vs.var('size').toString(), 20);
+
+    t.deepEquals(generatedQuery.body.score_functions, [
+      'focus_only_function'
+    ]);
+
+    t.deepEquals(generatedQuery.body.filter_functions, [
+      'boundary_country view',
+      'boundary_circle view',
+      'boundary_rect view',
+      'sources view'
+    ]);
+
+    t.deepEquals(logger.getInfoMessages(), ['[query:address_search_using_ids] [parser:libpostal]']);
+    t.end();
+
+  });
+};
+
+module.exports.tests.other_parameters = (test, common) => {
+  test('explicit size set', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      querySize: 'querySize value'
+    };
+    const res = {
+      data: []
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('size').toString(), 'querySize value');
+    t.deepEquals(logger.getInfoMessages(), ['[query:address_search_using_ids] [parser:libpostal] [param:querySize]']);
+    t.end();
+
+  });
+
+  test('explicit sources set', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      sources: ['source 1', 'source 2']
+    };
+    const res = {
+      data: []
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.deepEquals(generatedQuery.body.vs.var('sources').toString(), ['source 1', 'source 2']);
+    t.deepEquals(logger.getInfoMessages(), ['[query:address_search_using_ids] [parser:libpostal] [param:sources]']);
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.granularity_bands = (test, common) => {
+  test('neighbourhood/borough/locality/localadmin granularity band', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      }
+    };
+    const res = {
+      data: [
+        {
+          layer: 'neighbourhood',
+          source_id: 1
+        },
+        {
+          layer: 'borough',
+          source_id: 2
+        },
+        {
+          layer: 'locality',
+          source_id: 3
+        },
+        {
+          layer: 'localadmin',
+          source_id: 4
+        },
+        {
+          layer: 'county',
+          source_id: 5
+        },
+        {
+          layer: 'macrocounty',
+          source_id: 6
+        },
+        {
+          layer: 'region',
+          source_id: 7
+        },
+        {
+          layer: 'macroregion',
+          source_id: 8
+        },
+        {
+          layer: 'dependency',
+          source_id: 9
+        },
+        {
+          layer: 'country',
+          source_id: 10
+        },
+        {
+          layer: 'neighbourhood',
+          source_id: 11
+        },
+        {
+          layer: 'borough',
+          source_id: 12
+        },
+        {
+          layer: 'locality',
+          source_id: 13
+        },
+        {
+          layer: 'localadmin',
+          source_id: 14
+        },
+        {
+          layer: 'county',
+          source_id: 15
+        },
+        {
+          layer: 'macrocounty',
+          source_id: 16
+        },
+        {
+          layer: 'region',
+          source_id: 17
+        },
+        {
+          layer: 'macroregion',
+          source_id: 18
+        },
+        {
+          layer: 'dependency',
+          source_id: 19
+        },
+        {
+          layer: 'country',
+          source_id: 20
+        }
+      ]
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.deepEquals(generatedQuery.body.vs.var('input:layers').$, {
+      neighbourhood: [1, 11],
+      borough: [2, 12],
+      locality: [3, 13],
+      localadmin: [4, 14],
+      region: [7, 17],
+      macroregion: [8, 18],
+      dependency: [9, 19],
+      country: [10, 20]
+    });
+
+    t.end();
+  });
+
+  test('only band members with ids should be passed', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      }
+    };
+    const res = {
+      data: [
+        {
+          layer: 'neighbourhood',
+          source_id: 1
+        }
+      ]
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.deepEquals(generatedQuery.body.vs.var('input:layers').$, {
+      neighbourhood: [1],
+      borough: [],
+      locality: [],
+      localadmin: [],
+      region: [],
+      macroregion: [],
+      dependency: [],
+      country: []
+    });
+
+    t.end();
+  });
+
+  test('county/macrocounty granularity band', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      }
+    };
+    const res = {
+      data: [
+        {
+          layer: 'county',
+          source_id: 1
+        },
+        {
+          layer: 'macrocounty',
+          source_id: 2
+        },
+        {
+          layer: 'county',
+          source_id: 4
+        },
+        {
+          layer: 'macrocounty',
+          source_id: 5
+        }
+      ]
+    };
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.deepEquals(generatedQuery.body.vs.var('input:layers').$, {
+      county: [1, 4],
+      macrocounty: [2, 5]
+    });
+
+    t.end();
+  });
+
+};
+
+module.exports.tests.boundary_filters = (test, common) => {
+  test('boundary.country available should add to query', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      'boundary.country': 'boundary.country value'
+    };
+    const res = {};
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('boundary:country').toString(), 'boundary.country value');
+
+    t.end();
+
+  });
+
+  test('focus.point.lat/lon w/both numbers should add to query', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      'focus.point.lat': 12.121212,
+      'focus.point.lon': 21.212121
+    };
+    const res = {};
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('focus:point:lat').toString(), 12.121212);
+    t.equals(generatedQuery.body.vs.var('focus:point:lon').toString(), 21.212121);
+
+    t.end();
+
+  });
+
+  test('boundary.rect with all numbers should add to query', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      'boundary.rect.min_lat': 12.121212,
+      'boundary.rect.max_lat': 13.131313,
+      'boundary.rect.min_lon': 21.212121,
+      'boundary.rect.max_lon': 31.313131
+    };
+    const res = {};
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('boundary:rect:top').toString(), 13.131313);
+    t.equals(generatedQuery.body.vs.var('boundary:rect:right').toString(), 31.313131);
+    t.equals(generatedQuery.body.vs.var('boundary:rect:bottom').toString(), 12.121212);
+    t.equals(generatedQuery.body.vs.var('boundary:rect:left').toString(), 21.212121);
+
+    t.end();
+
+  });
+
+  test('boundary circle without radius should set radius to default', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      'boundary.circle.lat': 12.121212,
+      'boundary.circle.lon': 21.212121
+    };
+    const res = {};
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('boundary:circle:lat').toString(), 12.121212);
+    t.equals(generatedQuery.body.vs.var('boundary:circle:lon').toString(), 21.212121);
+    t.equals(generatedQuery.body.vs.var('boundary:circle:radius').toString(), '50km');
+
+    t.end();
+
+  });
+
+  test('boundary circle with radius set radius to that value rounded', (t) => {
+    const logger = mock_logger();
+
+    const clean = {
+      parsed_text: {
+        number: 'housenumber value',
+        street: 'street value'
+      },
+      'boundary.circle.lat': 12.121212,
+      'boundary.circle.lon': 21.212121,
+      'boundary.circle.radius': 17.6
+    };
+    const res = {};
+
+    const generateQuery = proxyquire('../../../query/address_search_using_ids', {
+      'pelias-logger': logger,
+      'pelias-query': {
+        layout: {
+          AddressesUsingIdsQuery: MockQuery
+        },
+        view: views,
+        Vars: require('pelias-query').Vars
+      }
+
+    });
+
+    const generatedQuery = generateQuery(clean, res);
+
+    t.equals(generatedQuery.body.vs.var('boundary:circle:lat').toString(), 12.121212);
+    t.equals(generatedQuery.body.vs.var('boundary:circle:lon').toString(), 21.212121);
+    t.equals(generatedQuery.body.vs.var('boundary:circle:radius').toString(), '18km');
+
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`address_search_using_ids query ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/search.js

@@ -284,7 +284,7 @@ module.exports.tests.city_state = function(test, common) {
 
     var query = generate(clean);
 
-    t.notEqual(query, undefined, 'should not have returned undefined');
+    t.equal(query, undefined, 'should have returned undefined');
     t.end();
 
   });
@@ -300,7 +300,7 @@ module.exports.tests.city_state = function(test, common) {
 
     var query = generate(clean);
 
-    t.notEqual(query, undefined, 'should not have returned undefined');
+    t.equal(query, undefined, 'should have returned undefined');
     t.end();
 
   });
@@ -458,7 +458,7 @@ module.exports.tests.city_country = function(test, common) {
 
     var query = generate(clean);
 
-    t.notEqual(query, undefined, 'should not have returned undefined');
+    t.equal(query, undefined, 'should have returned undefined');
     t.end();
 
   });
@@ -621,7 +621,6 @@ module.exports.tests.city_country = function(test, common) {
     t.end();
   });
 
-
 };
 
 module.exports.all = function (tape, common) {

test/unit/run.js

@@ -13,15 +13,23 @@ var tests = [
   require('./schema'),
   require('./controller/coarse_reverse'),
   require('./controller/index'),
+  require('./controller/libpostal'),
   require('./controller/place'),
   require('./controller/placeholder'),
   require('./controller/search'),
+  require('./controller/search_with_ids'),
+  require('./controller/predicates/has_parsed_text_properties'),
+  require('./controller/predicates/has_request_parameter'),
   require('./controller/predicates/has_response_data'),
   require('./controller/predicates/has_results_at_layers'),
+  require('./controller/predicates/has_request_parameter'),
   require('./controller/predicates/has_request_errors'),
+  require('./controller/predicates/is_addressit_parse'),
   require('./controller/predicates/is_admin_only_analysis'),
   require('./controller/predicates/is_coarse_reverse'),
-  require('./controller/predicates/is_service_enabled'),
+  require('./controller/predicates/is_only_non_admin_layers'),
+  require('./controller/predicates/is_request_sources_only_whosonfirst'),
+  require('./helper/debug'),
   require('./helper/diffPlaces'),
   require('./helper/geojsonify'),
   require('./helper/logging'),
@@ -45,6 +53,7 @@ var tests = [
   require('./middleware/trimByGranularity'),
   require('./middleware/trimByGranularityStructured'),
   require('./middleware/requestLanguage'),
+  require('./query/address_search_using_ids'),
   require('./query/autocomplete'),
   require('./query/autocomplete_defaults'),
   require('./query/search_defaults'),
@@ -55,6 +64,7 @@ var tests = [
   require('./query/structured_geocoding'),
   require('./query/text_parser'),
   require('./sanitizer/_boundary_country'),
+  require('./sanitizer/_debug'),
   require('./sanitizer/_flag_bool'),
   require('./sanitizer/_geonames_deprecation'),
   require('./sanitizer/_geonames_warnings'),
@@ -83,14 +93,14 @@ var tests = [
   require('./sanitizer/reverse'),
   require('./sanitizer/sanitizeAll'),
   require('./sanitizer/search'),
-  require('./sanitizer/search_fallback'),
+  require('./sanitizer/defer_to_addressit'),
   require('./sanitizer/wrap'),
+  require('./service/configurations/Interpolation'),
+  require('./service/configurations/Language'),
   require('./service/configurations/PlaceHolder'),
   require('./service/configurations/PointInPolygon'),
   require('./service/mget'),
-  require('./service/search'),
+  require('./service/search')
-  require('./service/interpolation'),
-  require('./service/language')
 ];
 
 tests.map(function(t) {

test/unit/sanitizer/_boundary_country.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_boundary_country');
+var sanitizer = require('../../../sanitizer/_boundary_country')();
 
 module.exports.tests = {};
 
@@ -6,7 +6,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('raw w/o boundary should set boundary.country undefined', function(t) {
     var raw = { };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], undefined, 'should be undefined');
     t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors');
     t.end();
@@ -15,7 +15,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('boundary.country explicitly undefined in raw should leave boundary.country undefined', function(t) {
     var raw = { 'boundary.country': undefined };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], undefined, 'should be undefined');
     t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors');
     t.end();
@@ -24,7 +24,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('non-string boundary.country should set boundary.country to undefined and return warning', function(t) {
     var raw = { 'boundary.country': ['this isn\'t a string primitive'] };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], undefined, 'should be undefined');
     t.deepEquals(errorsAndWarnings, { errors: ['boundary.country is not a string'], warnings: [] }, 'non-string country warning');
     t.end();
@@ -33,7 +33,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('iso2 boundary.country in raw should set boundary.country to ISO3 uppercased', function(t) {
     var raw = { 'boundary.country': 'aq' };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], 'ATA', 'should be uppercased ISO3');
     t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors');
     t.end();
@@ -42,7 +42,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('iso3 boundary.country in raw should set boundary.country to matching ISO3 uppercased', function(t) {
     var raw = { 'boundary.country': 'aTa' };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], 'ATA', 'should be uppercased ISO3');
     t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors');
     t.end();
@@ -51,7 +51,7 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('unknown 2-character boundary.country should set boundary.country to undefined', function(t) {
     var raw = { 'boundary.country': 'zq' };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], undefined, 'should be undefined');
     t.deepEquals(errorsAndWarnings, { errors: ['zq is not a valid ISO2/ISO3 country code'], warnings: [] }, 'country not found warning`');
     t.end();
@@ -60,17 +60,24 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
   test('unknown 3-character boundary.country should set boundary.country to undefined', function(t) {
     var raw = { 'boundary.country': 'zqx' };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var errorsAndWarnings = sanitizer.sanitize(raw, clean);
     t.equals(clean['boundary.country'], undefined, 'should be undefined');
     t.deepEquals(errorsAndWarnings, { errors: ['zqx is not a valid ISO2/ISO3 country code'], warnings: [] }, 'country not found warning`');
     t.end();
   });
 
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'boundary.country' }];
+    const validParameters = sanitizer.expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
+
 };
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _boundary_country ' + name, testFunction);
+    return tape('SANITIZE _boundary_country ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_categories.js

@@ -1,4 +1,4 @@
-var sanitize = require( '../../../sanitizer/_categories');
+var sanitizer = require( '../../../sanitizer/_categories')();
 
 module.exports.tests = {};
 
@@ -9,7 +9,7 @@ module.exports.tests.no_categories = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.equal(req.clean.categories, undefined, 'no categories should be defined');
     t.deepEqual(messages.errors, [], 'no error returned');
@@ -27,7 +27,7 @@ module.exports.tests.no_categories = function(test, common) {
 
     var expected_error = 'Categories parameter cannot be left blank. See documentation of service for valid options.';
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.equal(req.clean.categories, undefined, 'no categories should be defined');
     t.deepEqual(messages.errors.length, 1, 'error returned');
@@ -46,7 +46,7 @@ module.exports.tests.no_categories = function(test, common) {
 
     var expected_error = 'Invalid categories parameter value(s). See documentation of service for valid options.';
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.equal(req.clean.categories, undefined, 'no categories should be defined');
     t.deepEqual(messages.errors.length, 1, 'error returned');
@@ -74,7 +74,7 @@ module.exports.tests.valid_categories = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean, validCategories);
+    var messages = sanitizer.sanitize(req.query, req.clean, validCategories);
 
     t.deepEqual(req.clean.categories, ['food'], 'categories should contain food');
     t.deepEqual(messages.errors, [], 'no error returned');
@@ -95,7 +95,7 @@ module.exports.tests.valid_categories = function(test, common) {
     };
     var expectedCategories = ['food', 'health'];
 
-    var messages = sanitize(req.query, req.clean, validCategories);
+    var messages = sanitizer.sanitize(req.query, req.clean, validCategories);
 
     t.deepEqual(req.clean.categories, expectedCategories,
                 'clean.categories should be an array with proper values');
@@ -130,7 +130,7 @@ module.exports.tests.invalid_categories = function(test, common) {
       warnings: []
     };
 
-    var messages = sanitize(req.query, req.clean, validCategories);
+    var messages = sanitizer.sanitize(req.query, req.clean, validCategories);
 
     t.deepEqual(messages, expected_messages, 'error with message returned');
     t.equal(req.clean.categories, undefined, 'clean.categories should remain empty');
@@ -151,17 +151,26 @@ module.exports.tests.invalid_categories = function(test, common) {
       warnings: []
     };
 
-    var messages = sanitize(req.query, req.clean, validCategories);
+    var messages = sanitizer.sanitize(req.query, req.clean, validCategories);
 
     t.deepEqual(messages, expected_messages, 'error with message returned');
     t.equal(req.clean.categories, undefined, 'clean.categories should remain empty');
     t.end();
   });
+
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'categories' }];
+
+    const validParameters = sanitizer.expected();
+
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _categories ' + name, testFunction);
+    return tape('SANITIZE _categories ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_city_name_standardizer.js

@@ -1,5 +1,5 @@
 const _ = require('lodash');
-const sanitizer = require('../../../sanitizer/_city_name_standardizer');
+const sanitizer = require('../../../sanitizer/_city_name_standardizer')();
 
 module.exports.tests = {};
 
@@ -13,7 +13,7 @@ module.exports.tests.text_parser = function(test, common) {
     const expected_clean = {
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -39,7 +39,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -77,7 +77,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -115,7 +115,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -153,7 +153,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -191,7 +191,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -215,7 +215,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -239,7 +239,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -263,7 +263,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');

test/unit/sanitizer/_debug.js

@@ -0,0 +1,65 @@
+const sanitizer = require('../../../sanitizer/_debug')();
+
+module.exports.tests = {};
+
+module.exports.tests.sanitize_debug = function(test, common) {
+  ['true', '1', 1, true, 'TRUE', 'TrUe'].forEach((value) => {
+    test('debug flag is on', function(t) {
+      const raw = { debug: value };
+      const clean = {};
+      const expected_clean = { enableDebug: true  };
+
+      const messages = sanitizer.sanitize(raw, clean);
+
+      t.deepEquals(clean, expected_clean);
+      t.deepEqual(messages.errors, [], 'no error returned');
+      t.deepEqual(messages.warnings, [], 'no warnings returned');
+      t.end();
+    });
+  });
+
+  ['false', false, '0', 0, 'value', {}].forEach((value) => {
+    test('non-truthy values should set clean.debug to false', function(t) {
+      const raw = { debug: value };
+      const clean = {};
+      const expected_clean = { enableDebug: false  };
+
+      const messages = sanitizer.sanitize(raw, clean);
+
+      t.deepEquals(clean, expected_clean);
+      t.deepEqual(messages.errors, [], 'no error returned');
+      t.deepEqual(messages.warnings, [], 'no warnings returned');
+      t.end();
+    });
+  });
+
+  test('undefined default flag should not set clean.debug', function(t) {
+    const raw = {};
+    const clean = {};
+    const expected_clean = {};
+
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'debug' }];
+    const validParameters = sanitizer.expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANITIZE _debug ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitizer/_deprecate_quattroshapes.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_deprecate_quattroshapes');
+var sanitizer = require('../../../sanitizer/_deprecate_quattroshapes')();
 
 module.exports.tests = {};
 
@@ -7,7 +7,7 @@ module.exports.tests.warning_message_1 = function(test, common) {
     var raw = { sources: 'qs' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
     t.deepEquals(messages, {
       errors: [],
       warnings: ['You are using Quattroshapes as a data source in this query. ' +
@@ -27,7 +27,7 @@ module.exports.tests.warning_message_2 = function(test, common) {
     var raw = { sources: 'quattroshapes' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
     t.deepEquals(messages, {
       errors: [],
       warnings: ['You are using Quattroshapes as a data source in this query. ' +
@@ -47,7 +47,7 @@ module.exports.tests.rewrite = function(test, common) {
     var raw = { sources: 'qs,quattroshapes,qs,quattroshapes,osm' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
     t.equals(raw.sources,'osm,whosonfirst','use wof instead of qs');
 
     t.end();
@@ -56,7 +56,7 @@ module.exports.tests.rewrite = function(test, common) {
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _deprecate_quattroshapes ' + name, testFunction);
+    return tape('SANITIZE _deprecate_quattroshapes ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_flag_bool.js

@@ -1,5 +1,4 @@
 var sanitizer = require('../../../sanitizer/_flag_bool');
-var sanitize = sanitizer('dirty_param', true);
 
 module.exports.tests = {};
 
@@ -9,7 +8,7 @@ module.exports.tests.sanitize_private = function(test, common) {
     test('invalid dirty_param ' + value, function (t) {
       var raw = {dirty_param: value};
       var clean = {};
-      sanitize(raw, clean);
+      sanitizer('dirty_param', true).sanitize(raw, clean);
       t.equal(clean.dirty_param, false, 'default clean value set (to false)');
       t.end();
     });
@@ -20,7 +19,7 @@ module.exports.tests.sanitize_private = function(test, common) {
     test('valid dirty_param ' + value, function (t) {
       var raw = {dirty_param: value};
       var clean = {};
-      sanitize(raw, clean);
+      sanitizer('dirty_param', true).sanitize(raw, clean);
       t.equal(clean.dirty_param, true, 'clean value set to true');
       t.end();
     });
@@ -31,7 +30,7 @@ module.exports.tests.sanitize_private = function(test, common) {
     test('test setting false explicitly ' + value, function (t) {
       var raw = {dirty_param: value};
       var clean = {};
-      sanitize(raw, clean);
+      sanitizer('dirty_param', true).sanitize(raw, clean);
       t.equal(clean.dirty_param, false, 'clean value set to false');
       t.end();
     });
@@ -45,17 +44,26 @@ module.exports.tests.validate_default_behavior = function(test, common) {
       var sanitize_true = sanitizer('foo_bar', defaultValue);
       var raw = {};
       var clean = {};
-      sanitize_true(raw, clean);
+      sanitize_true.sanitize(raw, clean);
       t.equal(clean.foo_bar, defaultValue, 'foo_bar set to ' + defaultValue);
       t.end();
     });
   });
 };
 
+module.exports.tests.check_valid_parameters = function(test, common) {
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'value' }]; // depends on first argument of sanitizer()
+    const validParameters = sanitizer('value', true).expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE _flag_bool: ' + name, testFunction);
+    return tape('SANITIZE _flag_bool: ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_geo_reverse.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const sanitize = require('../../../sanitizer/_geo_reverse');
+const sanitizer = require('../../../sanitizer/_geo_reverse')();
 const defaults = require('../../../query/reverse_defaults');
 
 module.exports.tests = {};
@@ -13,7 +13,7 @@ module.exports.tests.warning_situations = (test, common) => {
       'boundary.circle.lat': '13.131313'
     };
     const clean = {};
-    const errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitizer.sanitize(raw, clean);
 
     t.equals(clean['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
     t.deepEquals(errorsAndWarnings, {
@@ -31,7 +31,7 @@ module.exports.tests.warning_situations = (test, common) => {
       'boundary.circle.lon': '31.313131'
     };
     const clean = {};
-    const errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitizer.sanitize(raw, clean);
 
     t.equals(clean['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
     t.deepEquals(errorsAndWarnings, {
@@ -49,7 +49,7 @@ module.exports.tests.warning_situations = (test, common) => {
       'boundary.circle.radius': '17'
     };
     const clean = {};
-    const errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitizer.sanitize(raw, clean);
 
     // t.equals(clean['boundary.circle.radius'], 12.121212, 'should be set to point.lat')
     t.deepEquals(errorsAndWarnings, {
@@ -70,7 +70,7 @@ module.exports.tests.success_conditions = (test, common) => {
       'boundary.circle.radius': '3248732857km' // this will never be the default
     };
     const clean = {};
-    const errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitizer.sanitize(raw, clean);
 
     t.equals(raw['boundary.circle.lat'], 12.121212);
     t.equals(raw['boundary.circle.lon'], 21.212121);

test/unit/sanitizer/_geonames_deprecation.js

@@ -1,4 +1,4 @@
-const geonames_deprecation = require('../../../sanitizer/_geonames_deprecation');
+const sanitizer = require('../../../sanitizer/_geonames_deprecation')();
 
 module.exports.tests = {};
 
@@ -6,7 +6,7 @@ module.exports.tests.no_warnings_or_errors_conditions = (test, common) => {
   test('undefined sources should add neither warnings nor errors', (t) => {
     const clean = {};
 
-    const messages = geonames_deprecation(undefined, clean);
+    const messages = sanitizer.sanitize(undefined, clean);
 
     t.deepEquals(clean, {});
     t.deepEquals(messages, { errors: [], warnings: [] });
@@ -19,7 +19,7 @@ module.exports.tests.no_warnings_or_errors_conditions = (test, common) => {
       sources: ['source 1', 'source 2'],
     };
 
-    const messages = geonames_deprecation(undefined, clean);
+    const messages = sanitizer.sanitize(undefined, clean);
 
     t.deepEquals(clean.sources, ['source 1', 'source 2']);
     t.deepEquals(messages, { errors: [], warnings: [] });
@@ -36,7 +36,7 @@ module.exports.tests.error_conditions = (test, common) => {
         sources: [sources]
       };
 
-      const messages = geonames_deprecation(undefined, clean);
+      const messages = sanitizer.sanitize(undefined, clean);
 
       t.deepEquals(clean.sources, [sources]);
       t.deepEquals(messages.errors, ['/reverse does not support geonames']);
@@ -57,7 +57,7 @@ module.exports.tests.warning_conditions = (test, common) => {
         sources: ['another source', sources, 'yet another source']
       };
 
-      const messages = geonames_deprecation(undefined, clean);
+      const messages = sanitizer.sanitize(undefined, clean);
 
       t.deepEquals(clean.sources, ['another source', 'yet another source']);
       t.deepEquals(messages.errors, []);
@@ -73,7 +73,7 @@ module.exports.tests.warning_conditions = (test, common) => {
 
 module.exports.all = (tape, common) => {
   function test(name, testFunction) {
-    return tape(`SANTIZE _geonames_deprecation ${name}`, testFunction);
+    return tape(`SANITIZE _geonames_deprecation ${name}`, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_geonames_warnings.js

@@ -1,6 +1,6 @@
 const _ = require('lodash');
 
-const geonames_warnings = require('../../../sanitizer/_geonames_warnings');
+const sanitizer = require('../../../sanitizer/_geonames_warnings')();
 
 const nonAdminProperties = ['number', 'street', 'query', 'category'];
 const adminProperties = ['neighbourhood', 'borough', 'city', 'county', 'state', 'postalcode', 'country'];
@@ -13,7 +13,7 @@ module.exports.tests.no_errors = (test, common) => {
       sources: ['geonames'],
     };
 
-    const messages = geonames_warnings(undefined, clean);
+    const messages = sanitizer.sanitize(undefined, clean);
 
     t.deepEquals(messages, { errors: [], warnings: [] });
     t.end();
@@ -30,7 +30,7 @@ module.exports.tests.no_errors = (test, common) => {
         clean.parsed_text[nonAdminProperty] = `${nonAdminProperty} value`;
         clean.parsed_text[adminProperty] = `${adminProperty} value`;
 
-        const messages = geonames_warnings(undefined, clean);
+        const messages = sanitizer.sanitize(undefined, clean);
 
         t.deepEquals(messages, { errors: [], warnings: [] });
 
@@ -50,7 +50,7 @@ module.exports.tests.no_errors = (test, common) => {
         clean.parsed_text[nonAdminProperty] = `${nonAdminProperty} value`;
         clean.parsed_text[adminProperty] = `${adminProperty} value`;
 
-        const messages = geonames_warnings(undefined, clean);
+        const messages = sanitizer.sanitize(undefined, clean);
 
         t.deepEquals(messages, { errors: [], warnings: [] });
 
@@ -68,7 +68,7 @@ module.exports.tests.error_conditions = (test, common) => {
       const clean = _.set({ sources: ['geonames'] },
         ['parsed_text', property], `${property} value`);
 
-      const messages = geonames_warnings(undefined, clean);
+      const messages = sanitizer.sanitize(undefined, clean);
 
       t.deepEquals(messages.errors, ['input contains only administrative area data, ' +
         'no results will be returned when sources=geonames']);
@@ -87,7 +87,7 @@ module.exports.tests.warning_conditions = (test, common) => {
       const clean = _.set({ sources: ['source 1', 'geonames', 'source 2'] },
         ['parsed_text', property], `${property} value`);
 
-      const messages = geonames_warnings(undefined, clean);
+      const messages = sanitizer.sanitize(undefined, clean);
 
       t.deepEquals(messages.errors, []);
       t.deepEquals(messages.warnings, ['input contains only administrative area data, ' +

test/unit/sanitizer/_ids.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_ids');
+var sanitizer = require('../../../sanitizer/_ids')();
 
 var delimiter = ':';
 var type_mapping = require('../../../helper/type_mapping');
@@ -15,7 +15,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: '' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], lengthError, 'ids length error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -26,7 +26,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: ':' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], formatError(':'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -37,7 +37,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: '::' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], formatError('::'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -48,7 +48,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: 'geoname:' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], formatError('geoname:'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -59,7 +59,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: ':234' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], formatError(':234'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -72,7 +72,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var expected_error = 'invalidsource is invalid. It must be one of these values - [' +
       Object.keys(type_mapping.source_mapping).join(', ') + ']';
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], expected_error, 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -83,7 +83,7 @@ module.exports.tests.invalid_ids = function(test, common) {
     var raw = { ids: 'geonames:23' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors[0], formatError('geonames:23'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
@@ -96,7 +96,7 @@ module.exports.tests.valid_ids = function(test, common) {
     var raw = { ids: 'openaddresses:address:20' };
     var clean = {};
 
-    var messages = sanitize( raw, clean );
+    var messages = sanitizer.sanitize( raw, clean );
 
     var expected_ids = [{
       source: 'openaddresses',
@@ -112,7 +112,7 @@ test('ids: valid short input (openaddresses)', function(t) {
     var raw = { ids: 'oa:address:20' };
     var clean = {};
 
-    var messages = sanitize( raw, clean );
+    var messages = sanitizer.sanitize( raw, clean );
 
     var expected_ids = [{
       source: 'openaddresses',
@@ -133,7 +133,7 @@ test('ids: valid short input (openaddresses)', function(t) {
       id: 'node:500',
     }];
 
-    var messages = sanitize( raw, clean );
+    var messages = sanitizer.sanitize( raw, clean );
 
     t.deepEqual( messages.errors, [], ' no errors');
     t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field');
@@ -149,7 +149,7 @@ test('ids: valid short input (openaddresses)', function(t) {
       id: 'node:500',
     }];
 
-    var messages = sanitize( raw, clean );
+    var messages = sanitizer.sanitize( raw, clean );
 
     t.deepEqual( messages.errors, [], ' no errors');
     t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field');
@@ -162,7 +162,7 @@ module.exports.tests.multiple_ids = function(test, common) {
     var raw = { ids: 'geonames:venue:1,openstreetmap:address:way:2' };
     var clean = {};
 
-    var messages = sanitize( raw, clean);
+    var messages = sanitizer.sanitize( raw, clean);
 
     var expected_ids = [ {
       source: 'geonames',
@@ -186,7 +186,7 @@ module.exports.tests.de_dupe = function(test, common) {
     var raw = { ids: 'geonames:venue:1,openstreetmap:venue:node:2,geonames:venue:1' };
     var clean = {};
 
-    var messages = sanitize( raw, clean );
+    var messages = sanitizer.sanitize( raw, clean );
 
     var expected_ids = [ {
       source: 'geonames',
@@ -204,9 +204,18 @@ module.exports.tests.de_dupe = function(test, common) {
   });
 };
 
+module.exports.tests.valid_Parameters = function(test, common) {
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'ids' }];
+    const validParameters = sanitizer.expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
+};
+
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _ids ' + name, testFunction);
+    return tape('SANITIZE _ids ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_iso2_to_iso3.js

@@ -1,4 +1,4 @@
-const sanitizer = require('../../../sanitizer/_iso2_to_iso3');
+const sanitizer = require('../../../sanitizer/_iso2_to_iso3')();
 
 module.exports.tests = {};
 
@@ -12,7 +12,7 @@ module.exports.tests.text_parser = function(test, common) {
     const expected_clean = {
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -38,7 +38,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -64,7 +64,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -90,7 +90,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -103,7 +103,7 @@ module.exports.tests.text_parser = function(test, common) {
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('sanitizer _iso2_to_iso3: ' + name, testFunction);
+    return tape('SANITIZE _iso2_to_iso3: ' + name, testFunction);
   }
 
   for( const testCase in module.exports.tests ){

test/unit/sanitizer/_layers.js

@@ -1,12 +1,12 @@
 var type_mapping = require('../../../helper/type_mapping');
 
-var sanitize = require('../../../sanitizer/_targets')('layers', type_mapping.layer_mapping);
+var sanitizer = require('../../../sanitizer/_targets')('layers', type_mapping.layer_mapping);
 
 module.exports.tests = {};
 
 module.exports.tests.sanitize_layers = function(test, common) {
   test('unspecified', function(t) {
-    var messages = sanitize({ layers: undefined }, {});
+    var messages = sanitizer.sanitize({ layers: undefined }, {});
     t.equal(messages.errors.length, 0, 'no errors');
     t.end();
   });
@@ -15,7 +15,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'test_layer' };
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     var msg = ' is an invalid layers parameter. Valid options: ';
     t.equal(messages.errors.length, 1, 'errors set');
@@ -28,7 +28,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'venue' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var venue_layers = ['venue'];
     t.deepEqual(clean.layers, venue_layers, 'venue layers set');
@@ -39,7 +39,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'coarse' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var admin_layers = [ 'continent', 'country', 'dependency',
     'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county',
@@ -53,7 +53,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'address' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     t.deepEqual(clean.layers, ['address'], 'address layer set');
     t.end();
@@ -63,7 +63,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'venue,country,region' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var expected_layers = ['venue', 'country', 'region'];
     t.deepEqual(clean.layers, expected_layers, 'venue + regular layers');
@@ -74,7 +74,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'coarse,country' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var expected_layers = [ 'continent', 'country', 'dependency',
     'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county',
@@ -88,7 +88,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'address,country,locality' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var expected_layers = ['address', 'country', 'locality' ];
     t.deepEqual(clean.layers, expected_layers, 'address + regular layers set');
@@ -99,7 +99,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'venue,country' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var expected_layers = ['venue', 'country'];
     t.deepEqual(clean.layers, expected_layers, 'venue layers found (no duplicates)');
@@ -110,7 +110,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var raw = { layers: 'venue,coarse' };
     var clean = {};
 
-    sanitize(raw, clean);
+    sanitizer.sanitize(raw, clean);
 
     var coarse_layers = [ 'continent',
       'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin',
@@ -126,7 +126,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _layers ' + name, testFunction);
+    return tape('SANITIZE _layers ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_location_bias.js

@@ -1,4 +1,4 @@
-const setup = require('../../../sanitizer/_location_bias');
+const sanitizer = require('../../../sanitizer/_location_bias');
 
 module.exports.tests = {};
 
@@ -8,14 +8,14 @@ module.exports.tests.setLocationBias = function(test, common) {
       'focus.point.lat': 12.12121212,
       'focus.point.lon': 21.21212121
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
     const raw = {};
     const expected = {
           'focus.point.lat': 12.12121212,
           'focus.point.lon': 21.21212121
     };
 
-    locationBias(raw, undefined);
+    locationBias.sanitize(raw, undefined);
     t.deepEqual(raw, expected, 'focus point should be set');
     t.end();
 
@@ -26,9 +26,9 @@ module.exports.tests.setLocationBias = function(test, common) {
         'focus.point.lat': 12.12121212,
         'focus.point.lon': 21.21212121
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
 
-    locationBias(undefined, undefined);
+    locationBias.sanitize(undefined, undefined);
     t.deepEqual(undefined, undefined, 'should be unmodified' );
     t.end();
   });
@@ -37,11 +37,11 @@ module.exports.tests.setLocationBias = function(test, common) {
     const defaultParameters = {
       'focus.point.lon': 12.2121212
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
     const raw = {};
     const expected = {};
 
-    locationBias(raw, undefined);
+    locationBias.sanitize(raw, undefined);
     t.deepEqual(raw, expected, 'should be unmodified' );
     t.end();
   });
@@ -50,11 +50,11 @@ module.exports.tests.setLocationBias = function(test, common) {
     const defaultParameters = {
       'focus.point.lat': 12.2121212
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
     const raw = {};
     const expected = {};
 
-    locationBias(raw, undefined);
+    locationBias.sanitize(raw, undefined);
       t.deepEqual(raw, expected, 'should be unmodified' );
       t.end();
   });
@@ -64,7 +64,7 @@ module.exports.tests.setLocationBias = function(test, common) {
       'focus.point.lon': 12.2121212,
       'focus.point.lat': 12.2121212
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
     const raw = {
       'focus.point.lon': 43.4343434
     };
@@ -72,7 +72,7 @@ module.exports.tests.setLocationBias = function(test, common) {
       'focus.point.lon': 43.4343434
     };
 
-    locationBias(raw, undefined);
+    locationBias.sanitize(raw, undefined);
       t.deepEqual(raw, expected, 'should be unmodified' );
       t.end();
   });
@@ -82,7 +82,7 @@ module.exports.tests.setLocationBias = function(test, common) {
       'focus.point.lon': 12.2121212,
       'focus.point.lat': 12.2121212
     };
-    const locationBias = setup(defaultParameters);
+    const locationBias = sanitizer(defaultParameters);
     const raw = {
       'focus.point.lat': 34.3434343
     };
@@ -90,7 +90,7 @@ module.exports.tests.setLocationBias = function(test, common) {
       'focus.point.lat': 34.3434343
     };
 
-    locationBias(raw, undefined);
+    locationBias.sanitize(raw, undefined);
       t.deepEqual(raw, expected, 'should be unmodified' );
       t.end();
   });

test/unit/sanitizer/_single_scalar_parameters.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_single_scalar_parameters');
+var sanitizer = require('../../../sanitizer/_single_scalar_parameters')();
 
 module.exports.tests = {};
 
@@ -10,8 +10,8 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
       arrayParameter2: ['value3']
     };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
-    t.deepEquals(errorsAndWarnings, {
+    t.deepEquals(messages, {
       errors: [
         '\'arrayParameter1\' parameter can only have one value',
         '\'arrayParameter2\' parameter can only have one value',
@@ -33,8 +33,8 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
       objectParameter2: { }
     };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
-    t.deepEquals(errorsAndWarnings, {
+    t.deepEquals(messages, {
       errors: [
         '\'objectParameter1\' parameter must be a scalar',
         '\'objectParameter2\' parameter must be a scalar'
@@ -51,8 +51,8 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
   test('request with all scalar parameters should return empty errors', function(t) {
     var raw = { scalarParameter1: 'value1', scalarParameter2: 2, scalarParameter3: true };
     var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
-    t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] });
+    t.deepEquals(messages, { errors: [], warnings: [] });
     t.end();
   });
 

test/unit/sanitizer/_size.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_size');
+var sanitizer = require('../../../sanitizer/_size');
 
 module.exports.tests = {};
 
@@ -6,7 +6,7 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size=0', function(t) {
     var raw = { size: 0 };
     var clean = {};
-    var res = sanitize(/*defaults*/)(raw, clean);
+    var res = sanitizer(/*defaults*/).sanitize(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 1, 'should return warning');
     t.equal(res.warnings[0], 'out-of-range integer \'size\', using MIN_SIZE', 'check warning text');
@@ -17,7 +17,7 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size=10000', function(t) {
     var raw = { size: 10000 };
     var clean = {};
-    var res = sanitize(/*defaults*/)(raw, clean);
+    var res = sanitizer(/*defaults*/).sanitize(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 1, 'should return warning');
     t.equal(res.warnings[0], 'out-of-range integer \'size\', using MAX_SIZE', 'check warning text');
@@ -28,20 +28,26 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size not set', function(t) {
     var raw = {};
     var clean = {};
-    var res = sanitize(/*defaults*/)(raw, clean);
+    var res = sanitizer(/*defaults*/).sanitize(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 0, 'should return no warning');
     t.equal(clean.size, 10, 'default to 10');
     t.end();
   });
 
+  test('return an array of expected parameters in object form for validation', function(t) {
+    const expected = [{ name: 'size' }];
+    const validParameters = sanitizer(/*defaults*/).expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
 
   var valid_sizes = [5, '5', 5.5, '5.5'];
   valid_sizes.forEach(function (size) {
     test('size=' + size, function (t) {
       var raw = {size: size};
       var clean = {};
-      var res = sanitize(/*defaults*/)(raw, clean);
+      var res = sanitizer(/*defaults*/).sanitize(raw, clean);
       t.equal(res.errors.length, 0, 'should return no errors');
       t.equal(res.warnings.length, 0, 'should return warning');
       t.equal(clean.size, 5, 'set to correct integer');
@@ -52,7 +58,7 @@ module.exports.tests.sanitize_size = function(test, common) {
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('SANTIZE _size ' + name, testFunction);
+    return tape('SANITIZE _size ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_sources.js

@@ -1,5 +1,5 @@
 var type_mapping = require('../../../helper/type_mapping');
-var sanitize = require( '../../../sanitizer/_targets' )('sources', type_mapping.source_mapping);
+var sanitizer = require( '../../../sanitizer/_targets' )('sources', type_mapping.source_mapping);
 
 var success_messages = { error: false };
 
@@ -12,7 +12,7 @@ module.exports.tests.no_sources = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.equal(req.clean.sources, undefined, 'no sources should be defined');
     t.deepEqual(messages.errors, [], 'no error returned');
@@ -31,7 +31,7 @@ module.exports.tests.no_sources = function(test, common) {
     var expected_error = 'sources parameter cannot be an empty string. ' +
        'Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst';
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.equal(req.clean.sources, undefined, 'no sources should be defined');
     t.deepEqual(messages.errors.length, 1, 'error returned');
@@ -50,7 +50,7 @@ module.exports.tests.valid_sources = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.deepEqual(req.clean.sources, ['geonames'], 'sources should contain geonames');
     t.deepEqual(messages.errors, [], 'no error returned');
@@ -67,7 +67,7 @@ module.exports.tests.valid_sources = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.deepEqual(req.clean.sources, ['openstreetmap'], 'abbreviation is expanded to full version');
     t.deepEqual(messages.errors, [], 'no error returned');
@@ -83,7 +83,7 @@ module.exports.tests.valid_sources = function(test, common) {
       clean: { }
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.deepEqual(req.clean.sources, ['openstreetmap', 'openaddresses'],
                 'clean.sources should contain openstreetmap and openadresses');
@@ -109,7 +109,7 @@ module.exports.tests.invalid_sources = function(test, common) {
       warnings: []
     };
 
-    var messages = sanitize(req.query, req.clean);
+    var messages = sanitizer.sanitize(req.query, req.clean);
 
     t.deepEqual(messages, expected_messages, 'error with message returned');
     t.equal(req.clean.sources, undefined, 'clean.sources should remain empty');

test/unit/sanitizer/_sources_and_layers.js

@@ -1,4 +1,4 @@
-var sanitize = require('../../../sanitizer/_sources_and_layers');
+var sanitizer = require('../../../sanitizer/_sources_and_layers')();
 
 var type_mapping = require('../../../helper/type_mapping');
 
@@ -9,7 +9,7 @@ module.exports.tests.inactive = function(test, common) {
     var raw = {};
     var clean = {};
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -20,7 +20,7 @@ module.exports.tests.inactive = function(test, common) {
     var raw = {};
     var clean = { layers: ['venue'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -31,7 +31,7 @@ module.exports.tests.inactive = function(test, common) {
     var raw = {};
     var clean = { sources: ['openstreetmap'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -44,7 +44,7 @@ module.exports.tests.no_errors = function(test, common) {
     var raw = {};
     var clean = { sources: ['openstreetmap'], layers: ['venue'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -55,7 +55,7 @@ test('valid combination', function(t) {
     var raw = {};
     var clean = { sources: ['geonames'], layers: ['borough'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -66,7 +66,7 @@ test('valid combination', function(t) {
     var raw = {};
     var clean = { sources: ['geonames'], layers: ['macroregion'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -77,7 +77,7 @@ test('valid combination', function(t) {
     var raw = {};
     var clean = { sources: ['whosonfirst'], layers: ['venue'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -88,7 +88,7 @@ test('valid combination', function(t) {
     var raw = {};
     var clean = { sources: ['openstreetmap', 'openaddresses'], layers: ['venue'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
@@ -99,12 +99,19 @@ test('valid combination', function(t) {
     var raw = {};
     var clean = { sources: ['openaddresses'], layers: ['address', 'country'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 0, 'should return no errors');
     t.equal(messages.warnings.length, 0, 'should return no warnings');
     t.end();
   });
+
+  test('return an array of expected parameters in object form for validation', function (t) {
+    const expected = [{ 'name': 'sources' }, { 'name': 'layers' }];
+    const validParameters = sanitizer.expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
 };
 
 module.exports.tests.invalid_combination = function(test, common) {
@@ -112,7 +119,7 @@ module.exports.tests.invalid_combination = function(test, common) {
     var raw = {};
     var clean = { sources: ['whosonfirst'], layers: ['address'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 1, 'should return an error');
     t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' +
@@ -125,7 +132,7 @@ module.exports.tests.invalid_combination = function(test, common) {
     var raw = {};
     var clean = { sources: ['openstreetmap'], layers: ['country', 'locality'] };
 
-    var messages = sanitize(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.equal(messages.errors.length, 2, 'should return an error');
     t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' +

test/unit/sanitizer/_synthesize_analysis.js

@@ -37,7 +37,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -76,7 +76,7 @@ module.exports.tests.text_parser = function(test, common) {
       parsed_text: {}
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, ['at least one of the following fields is required: ' +
@@ -99,7 +99,7 @@ module.exports.tests.text_parser = function(test, common) {
 
     const expected_clean = { parsed_text: {} };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, ['at least one of the following fields is required: ' +
@@ -128,7 +128,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -161,7 +161,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -194,7 +194,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -224,7 +224,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    const messages = sanitizer(raw, clean);
+    const messages = sanitizer().sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
@@ -233,6 +233,23 @@ module.exports.tests.text_parser = function(test, common) {
 
   });
 
+  test('return an array of expected parameters in object form for validation', function (t) {
+    const sanitizer = require('../../../sanitizer/_synthesize_analysis');
+    const expected = [
+      { 'name': 'venue' },
+      { 'name': 'address' },
+      { 'name': 'neighbourhood' },
+      { 'name': 'borough' },
+      { 'name': 'locality' },
+      { 'name': 'county' },
+      { 'name': 'region' },
+      { 'name': 'postalcode' },
+      { 'name': 'country' }];
+
+    const validParameters = sanitizer().expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/sanitizer/_text.js

@@ -1,152 +1,62 @@
-var type_mapping = require('../../../helper/type_mapping');
+const sanitizer = require('../../../sanitizer/_text')();
-var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
 
 module.exports.tests.text_parser = function(test, common) {
-  test('non-empty raw.text should call analyzer and set clean.text and clean.parsed_text', function(t) {
+  test('non-empty raw.text should overwrite clean.text', t => {
-    var mock_analyzer_response = {
+    const raw = {
-      key1: 'value 1',
-      key2: 'value 2'
-    };
-
-    var sanitizer = proxyquire('../../../sanitizer/_text', {
-      'pelias-text-analyzer': { parse: function(query) {
-        return mock_analyzer_response;
-      }
-    }});
-
-    var raw = {
       text: 'raw input'
     };
-    var clean = {
+    const clean = {
+      text: 'original clean.text'
     };
 
-    var expected_clean = {
+    const expected_clean = {
-      text: raw.text,
+      text: raw.text
-      parsed_text: mock_analyzer_response
     };
 
-    var messages = sanitizer(raw, clean);
+    const messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(clean, expected_clean);
-    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages, { warnings: [], errors: [] }, 'no errors/warnings');
-    t.deepEquals(messages.warnings, [], 'no warnings');
     t.end();
 
   });
 
-  test('empty raw.text should add error message', function(t) {
+  test('undefined/empty raw.text should add error message', t => {
-    var sanitizer = proxyquire('../../../sanitizer/_text', {
+    [undefined, ''].forEach(val => {
-      'pelias-text-analyzer': { parse: function(query) {
+      const raw = {
-        throw new Error('analyzer should not have been called');
+        text: val
-      }
-    }});
-
-    var raw = {
-      text: ''
       };
-    var clean = {
+      const clean = {
       };
 
-    var expected_clean = {
+      const expected_clean = {
       };
 
-    var messages = sanitizer(raw, clean);
+      const messages = sanitizer.sanitize(raw, clean);
 
       t.deepEquals(clean, expected_clean);
       t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors');
       t.deepEquals(messages.warnings, [], 'no warnings');
-    t.end();
 
     });
 
-  test('undefined raw.text should add error message', function(t) {
-    var sanitizer = proxyquire('../../../sanitizer/_text', {
-      'pelias-text-analyzer': { parse: function(query) {
-        throw new Error('analyzer should not have been called');
-      }
-    }});
-
-    var raw = {
-      text: undefined
-    };
-    var clean = {
-    };
-
-    var expected_clean = {
-    };
-
-    var messages = sanitizer(raw, clean);
-
-    t.deepEquals(clean, expected_clean);
-    t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors');
-    t.deepEquals(messages.warnings, [], 'no warnings');
     t.end();
 
   });
 
-  test('text_analyzer.parse returning undefined should not overwrite clean.parsed_text', function(t) {
+  test('return an array of expected parameters in object form for validation', (t) => {
-    var sanitizer = proxyquire('../../../sanitizer/_text', {
+    const expected = [{ name: 'text' }];
-      'pelias-text-analyzer': { parse: function(query) {
+    const validParameters = sanitizer.expected();
-        return undefined;
+    t.deepEquals(validParameters, expected);
-      }
-    }});
-
-    var raw = {
-      text: 'raw input'
-    };
-    var clean = {
-      parsed_text: 'original clean.parsed_text'
-    };
-
-    var expected_clean = {
-      text: raw.text,
-      parsed_text: 'original clean.parsed_text'
-    };
-
-    var messages = sanitizer(raw, clean);
-
-    t.deepEquals(clean, expected_clean);
-    t.deepEquals(messages.errors, [], 'no errors');
-    t.deepEquals(messages.warnings, [], 'no warnings');
     t.end();
-
   });
-
-  test('text_analyzer.parse returning null should not overwrite clean.parsed_text', function(t) {
-    var sanitizer = proxyquire('../../../sanitizer/_text', {
-      'pelias-text-analyzer': { parse: function(query) {
-        return null;
-      }
-    }});
-
-    var raw = {
-      text: 'raw input'
-    };
-    var clean = {
-      parsed_text: 'original clean.parsed_text'
-    };
-
-    var expected_clean = {
-      text: raw.text,
-      parsed_text: 'original clean.parsed_text'
-    };
-
-    var messages = sanitizer(raw, clean);
-
-    t.deepEquals(clean, expected_clean);
-    t.deepEquals(messages.errors, [], 'no errors');
-    t.deepEquals(messages.warnings, [], 'no warnings');
-    t.end();
-
-  });
-
 };
 
-module.exports.all = function (tape, common) {
+module.exports.all = (tape, common) => {
   function test(name, testFunction) {
-    return tape('sanitizeR _text: ' + name, testFunction);
+    return tape(`sanitizer _text: ${name}`, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/_text_addressit.js

@@ -1,4 +1,4 @@
-var sanitizer = require('../../../sanitizer/_text_addressit');
+var sanitizer = require('../../../sanitizer/_text_addressit')();
 var type_mapping = require('../../../helper/type_mapping');
 
 module.exports.tests = {};
@@ -11,7 +11,7 @@ module.exports.tests.text_parser = function(test, common) {
     var clean = {
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEquals(messages.errors, [], 'no errors');
     t.deepEquals(messages.warnings, [], 'no warnings');
@@ -33,6 +33,7 @@ module.exports.tests.text_parser = function(test, common) {
 
       var expected_clean = {
         text: query.name + ', ' + query.admin_parts,
+        parser: 'addressit',
         parsed_text: {
           name: query.name,
           regions: [ query.name ],
@@ -41,7 +42,7 @@ module.exports.tests.text_parser = function(test, common) {
         }
       };
 
-      var messages = sanitizer(raw, clean);
+      var messages = sanitizer.sanitize(raw, clean);
 
       t.deepEqual(messages, { errors: [], warnings: [] } );
       t.deepEqual(clean, expected_clean);
@@ -57,6 +58,7 @@ module.exports.tests.text_parser = function(test, common) {
 
       var expected_clean = {
         text: query.name + ',' + query.admin_parts,
+        parser: 'addressit',
         parsed_text: {
           name: query.name,
           regions: [ query.name ],
@@ -65,7 +67,7 @@ module.exports.tests.text_parser = function(test, common) {
         }
       };
 
-      var messages = sanitizer(raw, clean);
+      var messages = sanitizer.sanitize(raw, clean);
 
       t.deepEqual(messages, { errors: [], warnings: [] } );
       t.deepEqual(clean, expected_clean);
@@ -88,6 +90,7 @@ module.exports.tests.text_parser = function(test, common) {
 
       var expected_clean = {
         text: query.name + ', ' + query.admin_parts,
+        parser: 'addressit',
         parsed_text: {
           name: query.name,
           regions: [ query.name, query.admin_parts ],
@@ -95,7 +98,7 @@ module.exports.tests.text_parser = function(test, common) {
         }
       };
 
-      var messages = sanitizer(raw, clean);
+      var messages = sanitizer.sanitize(raw, clean);
 
       t.deepEqual(messages, { errors: [], warnings: [] } );
       t.deepEqual(clean, expected_clean);
@@ -111,6 +114,7 @@ module.exports.tests.text_parser = function(test, common) {
 
       var expected_clean = {
         text: query.name + ',' + query.admin_parts,
+        parser: 'addressit',
         parsed_text: {
           name: query.name,
           regions: [ query.name, query.admin_parts ],
@@ -118,7 +122,7 @@ module.exports.tests.text_parser = function(test, common) {
         }
       };
 
-      var messages = sanitizer(raw, clean);
+      var messages = sanitizer.sanitize(raw, clean);
 
       t.deepEqual(messages, { errors: [], warnings: [] } );
       t.deepEqual(clean, expected_clean);
@@ -136,10 +140,11 @@ module.exports.tests.text_parser = function(test, common) {
     clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
+      parser: 'addressit',
       text: 'yugolsavia'
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -155,10 +160,11 @@ module.exports.tests.text_parser = function(test, common) {
     clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
+      parser: 'addressit',
       text: 'small town'
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -174,10 +180,11 @@ module.exports.tests.text_parser = function(test, common) {
     clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
+      parser: 'addressit',
       text: '123 main'
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -193,10 +200,11 @@ module.exports.tests.text_parser = function(test, common) {
     clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
+      parser: 'addressit',
       text: 'main 123'
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -213,13 +221,14 @@ module.exports.tests.text_parser = function(test, common) {
 
     var expected_clean = {
       text: 'main particle new york',
+      parser: 'addressit',
       parsed_text: {
         regions: [ 'main particle' ],
         state: 'NY'
       }
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -235,6 +244,7 @@ module.exports.tests.text_parser = function(test, common) {
 
     var expected_clean = {
       text: '123 main st new york ny',
+      parser: 'addressit',
       parsed_text: {
         number: '123',
         street: 'main st',
@@ -243,7 +253,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -259,6 +269,7 @@ module.exports.tests.text_parser = function(test, common) {
 
     var expected_clean = {
       text: '123 main st new york ny 10010',
+      parser: 'addressit',
       parsed_text: {
         number: '123',
         street: 'main st',
@@ -268,7 +279,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -283,6 +294,7 @@ module.exports.tests.text_parser = function(test, common) {
 
     var expected_clean = {
       text: '339 W Main St, Cheshire, 06410',
+      parser: 'addressit',
       parsed_text: {
         name: '339 W Main St',
         number: '339',
@@ -293,7 +305,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -308,6 +320,7 @@ module.exports.tests.text_parser = function(test, common) {
 
     var expected_clean = {
       text: '339 W Main St,Lancaster,PA',
+      parser: 'addressit',
       parsed_text: {
         name: '339 W Main St',
         number: '339',
@@ -318,7 +331,7 @@ module.exports.tests.text_parser = function(test, common) {
       }
     };
 
-    var messages = sanitizer(raw, clean);
+    var messages = sanitizer.sanitize(raw, clean);
 
     t.deepEqual(messages, { errors: [], warnings: [] } );
     t.deepEqual(clean, expected_clean);
@@ -326,6 +339,12 @@ module.exports.tests.text_parser = function(test, common) {
 
   });
 
+  test('return an array of expected parameters in object form for validation', (t) => {
+    const expected = [{ name: 'text' }];
+    const validParameters = sanitizer.expected();
+    t.deepEquals(validParameters, expected);
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/sanitizer/_tokenizer.js

@@ -1,4 +1,4 @@
-var sanitizer = require('../../../sanitizer/_tokenizer');
+var sanitizer = require('../../../sanitizer/_tokenizer')();
 
 module.exports.tests = {};
 
@@ -6,7 +6,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('clean.text not set', function(t) {
 
     var clean = {}; // clean.text not set
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // no tokens produced
     t.deepEquals(clean.tokens, [], 'no tokens');
@@ -22,7 +22,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('clean.text not a string', function(t) {
 
     var clean = { text: {} }; // clean.text not a string
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // no tokens produced
     t.deepEquals(clean.tokens, [], 'no tokens');
@@ -38,7 +38,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('empty string', function(t) {
 
     var clean = { text: '' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // no tokens produced
     t.deepEquals(clean.tokens, [], 'no tokens');
@@ -54,7 +54,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('clean.parsed_text set but clean.parsed_text.name invalid', function(t) {
 
     var clean = { parsed_text: { text: {} } };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // no tokens produced
     t.deepEquals(clean.tokens, [], 'no tokens');
@@ -70,7 +70,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('favor clean.parsed_text.name over clean.text', function(t) {
 
     var clean = { parsed_text: { name: 'foo' }, text: 'bar' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // favor clean.parsed_text.name over clean.text
     t.deepEquals(clean.tokens, [ 'foo' ], 'use clean.parsed_text.name');
@@ -86,7 +86,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('favor clean.parsed_text street data over clean.text', function(t) {
 
     var clean = { parsed_text: { number: '190', street: 'foo st' }, text: 'bar' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // favor clean.parsed_text.name over clean.text
     t.deepEquals(clean.tokens, [ '190', 'foo', 'st' ], 'use street name + number');
@@ -102,7 +102,7 @@ module.exports.tests.sanity_checks = function(test, common) {
   test('favor clean.parsed_text.name over clean.parsed_text street data', function(t) {
 
     var clean = { parsed_text: { number: '190', street: 'foo st', name: 'foo' }, text: 'bar' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // favor clean.parsed_text.name over all other variables
     t.deepEquals(clean.tokens, [ 'foo' ], 'use clean.parsed_text.name');
@@ -121,7 +121,7 @@ module.exports.tests.space_delimiter = function(test, common) {
   test('space delimiter - simple', function(t) {
 
     var clean = { text: '30 west 26th street new york' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -156,7 +156,7 @@ module.exports.tests.space_delimiter = function(test, common) {
   test('space delimiter - multiple spaces / other whitespace', function(t) {
 
     var clean = { text: ' 30  west \t26th \nstreet   new york ' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -194,7 +194,7 @@ module.exports.tests.comma_delimiter = function(test, common) {
   test('comma delimiter - simple', function(t) {
 
     var clean = { text: '30 west 26th street, new york' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -229,7 +229,7 @@ module.exports.tests.comma_delimiter = function(test, common) {
   test('comma delimiter - multiple commas', function(t) {
 
     var clean = { text: ',30 west 26th street,,, new york,' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -267,7 +267,7 @@ module.exports.tests.forward_slash_delimiter = function(test, common) {
   test('forward slash delimiter - simple', function(t) {
 
     var clean = { text: 'Bedell Street/133rd Avenue' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -298,7 +298,7 @@ module.exports.tests.forward_slash_delimiter = function(test, common) {
   test('forward slash - multiple slashes', function(t) {
 
     var clean = { text: '/Bedell Street//133rd Avenue/' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -332,7 +332,7 @@ module.exports.tests.final_token_single_gram = function(test, common) {
   test('final token single gram - numeric', function(t) {
 
     var clean = { text: 'grolmanstrasse 1' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -359,7 +359,7 @@ module.exports.tests.final_token_single_gram = function(test, common) {
   test('final token single gram - non-numeric', function(t) {
 
     var clean = { text: 'grolmanstrasse a' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -389,7 +389,7 @@ module.exports.tests.back_slash_delimiter = function(test, common) {
   test('back slash delimiter - simple', function(t) {
 
     var clean = { text: 'Bedell Street\\133rd Avenue' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -408,7 +408,7 @@ module.exports.tests.back_slash_delimiter = function(test, common) {
   test('back slash - multiple slashes', function(t) {
 
     var clean = { text: '\\Bedell Street\\\\133rd Avenue\\' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [
@@ -430,7 +430,7 @@ module.exports.tests.mixed_delimiter = function(test, common) {
   test('mixed delimiters', function(t) {
 
     var clean = { text: ',/Bedell Street\\, \n\t ,\\//133rd Avenue, /\n/' };
-    var messages = sanitizer({}, clean);
+    var messages = sanitizer.sanitize({}, clean);
 
     // tokens produced
     t.deepEquals(clean.tokens, [

test/unit/sanitizer/autocomplete.js

@@ -8,86 +8,124 @@ module.exports.tests.sanitizers = function(test, common) {
     var called_sanitizers = [];
 
     var autocomplete = proxyquire('../../../sanitizer/autocomplete', {
-      '../sanitizer/_single_scalar_parameters': () => {
+      '../sanitizer/_single_scalar_parameters': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_single_scalar_parameters');
             return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_debug');
+            return { errors: [], warnings: [] };
+          }
+        };
       },
-      '../sanitizer/_text_addressit': () => {
+      '../sanitizer/_text_addressit': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_text_addressit');
             return { errors: [], warnings: [] };
+          }
+        };
       },
-      '../sanitizer/_tokenizer': () => {
+      '../sanitizer/_tokenizer': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_tokenizer');
             return { errors: [], warnings: [] };
+          }
+        };
       },
       '../sanitizer/_size': function () {
         if (_.isEqual(_.values(arguments), [10, 10, 10])) {
-          return () => {
+          return {
+            sanitize: () => {
               called_sanitizers.push('_size');
               return { errors: [], warnings: [] };
+              }
           };
-
         } else {
           throw new Error('incorrect parameters passed to _size');
         }
-
       },
-      '../sanitizer/_targets': (type) => {
+      '../sanitizer/_targets': function (type) {
         if (['layers', 'sources'].indexOf(type) !== -1) {
-          return () => {
+          return {
+            sanitize: () => {
               called_sanitizers.push(`_targets/${type}`);
               return { errors: [], warnings: [] };
-          };
-
               }
-        else {
+            };
+          } else {
           throw new Error('incorrect parameters passed to _targets');
         }
-
       },
-      '../sanitizer/_sources_and_layers': () => {
+
+      '../sanitizer/_sources_and_layers': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_sources_and_layers');
             return { errors: [], warnings: [] };
+          }
+        };
       },
       '../sanitizer/_flag_bool': function () {
         if (arguments[0] === 'private' && arguments[1] === false) {
-          return () => {
+          return {
+            sanitize: () => {
                 called_sanitizers.push('_flag_bool');
                 return { errors: [], warnings: [] };
-          };
-
               }
-        else {
+            };
+        } else {
             throw new Error('incorrect parameters passed to _flag_bool');
         }
-
       },
-      '../sanitizer/_location_bias': (defaultParameters) => {
+      '../sanitizer/_location_bias': function (defaultParameters) {
+        return {
+          sanitize: () => {
             if (defaultParameters.key === 'value'){
-          return () => {
                 called_sanitizers.push('_location_bias');
                 return { errors: [], warnings: [] };
-          };
             } else {
                 throw new Error('incorrect parameter passed to _location_bias');
             }
+          }
+        };
       },
-      '../sanitizer/_geo_autocomplete': () => {
+      '../sanitizer/_geo_autocomplete': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_geo_autocomplete');
             return { errors: [], warnings: [] };
+          }
+        };
       },
-      '../sanitizer/_boundary_country': () => {
+      '../sanitizer/_boundary_country': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_boundary_country');
             return { errors: [], warnings: [] };
+          }
+        };
       },
-      '../sanitizer/_categories': () => {
+      '../sanitizer/_categories': function () {
+        return {
+          sanitize: () => {
             called_sanitizers.push('_categories');
             return { errors: [], warnings: [] };
-      },
+          }
+        };
+      }
     });
 
     const expected_sanitizers = [
       '_single_scalar_parameters',
+      '_debug',
       '_text_addressit',
       '_tokenizer',
       '_size',

test/unit/sanitizer/defer_to_addressit.js

@@ -0,0 +1,179 @@
+const proxyquire =  require('proxyquire').noCallThru();
+const mock_logger = require('pelias-mock-logger');
+
+module.exports.tests = {};
+
+module.exports.tests.sanitize = (test, common) => {
+  test('verify that no sanitizers were called when should_execute returns false', (t) => {
+    t.plan(1);
+
+    const logger = mock_logger();
+
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
+    //  were all called correctly
+    const defer_to_addressit = proxyquire('../../../sanitizer/defer_to_addressit', {
+      '../sanitizer/_text_addressit': function () {
+        return {
+          sanitize: () => {
+            t.fail('_text_addressit should not have been called');
+          }
+        };
+      },
+      'pelias-logger': logger,
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            t.fail('_debug should not have been called');
+          }
+        };
+      }
+    })(() => false);
+
+    defer_to_addressit({}, {}, () => {
+      t.equals(logger.getInfoMessages().length, 0);
+      t.end();
+    });
+
+  });
+
+  test('verify that _text_addressit sanitizer was called when should_execute returns true', (t) => {
+    t.plan(3);
+
+    const logger = mock_logger();
+
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
+    //  were all called correctly
+    const defer_to_addressit = proxyquire('../../../sanitizer/defer_to_addressit', {
+      '../sanitizer/_text_addressit': function () {
+        return {
+          sanitize: () => {
+            t.pass('_text_addressit should have been called');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      'pelias-logger': logger,
+      '../helper/logging': {
+        isDNT: () => false
+      },
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            t.pass('_debug should have been called');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+    })(() => true);
+
+    const req = {
+      path: '/v1/search',
+      clean: {
+        text: 'this is the query text'
+      }
+    };
+
+    defer_to_addressit(req, {}, () => {
+      t.deepEquals(logger.getInfoMessages(), ['fallback queryText: this is the query text']);
+      t.end();
+    });
+
+  });
+
+  test('query should not be logged if path != \'/v1/search\'', (t) => {
+    t.plan(3);
+
+    const logger = mock_logger();
+
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
+    //  were all called correctly
+    const defer_to_addressit = proxyquire('../../../sanitizer/defer_to_addressit', {
+      '../sanitizer/_text_addressit': function () {
+        return {
+          sanitize: () => {
+            t.pass('_text_addressit should have been called');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      'pelias-logger': logger,
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            t.pass('_debug should have been called');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+    })(() => true);
+
+    const req = {
+      path: 'not /v1/search',
+      clean: {
+        text: 'this is the query text'
+      }
+    };
+
+    defer_to_addressit(req, {}, () => {
+      t.deepEquals(logger.getInfoMessages(), []);
+      t.end();
+    });
+
+  });
+
+  test('query should be logged as [text removed] if private', (t) => {
+    t.plan(3);
+
+    const logger = mock_logger();
+
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
+    //  were all called correctly
+    const defer_to_addressit = proxyquire('../../../sanitizer/defer_to_addressit', {
+      '../sanitizer/_text_addressit': function () {
+        return {
+          sanitize: () => {
+            t.pass('_text_addressit should have been called');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      'pelias-logger': logger,
+      '../helper/logging': {
+        isDNT: () => true
+      },
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            t.pass('_debug should have been called');
+          return { errors: [], warnings: [] };
+          }
+        };
+      }
+    })(() => true);
+
+    const req = {
+      path: '/v1/search',
+      clean: {
+        text: 'this is the query text'
+      }
+    };
+
+    defer_to_addressit(req, {}, () => {
+      t.deepEquals(logger.getInfoMessages(), ['fallback queryText: [text removed]']);
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape(`SANITIZE /defer_to_addressit ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitizer/nearby.js

@@ -1,58 +1,146 @@
+const  _ = require('lodash'),
+    proxyquire =  require('proxyquire').noCallThru();
 
-var nearby  = require('../../../sanitizer/nearby');
+module.exports.tests = {};
-var defaults = require('../../../query/reverse_defaults');
+
-var sanitize = nearby.sanitize;
+module.exports.tests.sanitize = function(test, common) {
-var middleware = nearby.middleware;
+  test('verify that all sanitizers were called as expected', function(t) {
+    var called_sanitizers = [];
 
-var defaultClean =  { 'point.lat': 0,
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
-                      'point.lon': 0,
+    //  were all called correctly
-                      'boundary.circle.lat': 0,
+    var nearby = proxyquire('../../../sanitizer/nearby.js', {
-                      'boundary.circle.lon': 0,
+      '../sanitizer/_single_scalar_parameters': function () {
-                      size: 10,
+        return {
-                      private: false
+          sanitize: () => {
+            called_sanitizers.push('_single_scalar_parameters');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_debug': () => {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_debug');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_deprecate_quattroshapes': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_deprecate_quattroshapes');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_targets': function (type) {
+        if (['layers', 'sources'].indexOf(type) !== -1) {
+          return {
+            sanitize: () => {
+              called_sanitizers.push(`_targets/${type}`);
+              return { errors: [], warnings: [] };
+              }
+            };
+          } else {
+          throw new Error('incorrect parameters passed to _targets');
+        }
+      },
+      '../sanitizer/_sources_and_layers': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_sources_and_layers');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_geonames_deprecation': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_geonames_deprecations');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_size': function () {
+        if (_.isEmpty(arguments)) {
+          return {
+            sanitize: () => {
+              called_sanitizers.push('_size');
+              return { errors: [], warnings: [] };
+              }
+          };
+        } else {
+          throw new Error('should not have passed any parameters to _size');
+        }
+      },
+      '../sanitizer/_flag_bool': function () {
+        if (arguments[0] === 'private' && arguments[1] === false) {
+          return {
+            sanitize: () => {
+                called_sanitizers.push('_flag_bool');
+                return { errors: [], warnings: [] };
+              }
             };
+        } else {
+            throw new Error('incorrect parameters passed to _flag_bool');
+        }
+      },
+      '../sanitizer/_geo_reverse': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_geo_reverse');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_boundary_country': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_boundary_country');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_categories': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_categories');
+            return { errors: [], warnings: [] };
+          }
+        };
+      }
+    });
 
-module.exports.tests = {};
+    const expected_sanitizers = [
+      '_single_scalar_parameters',
+      '_debug',
+      '_deprecate_quattroshapes',
+      '_targets/layers',
+      '_targets/sources',
+      '_sources_and_layers',
+      '_geonames_deprecations',
+      '_size',
+      '_flag_bool',
+      '_geo_reverse',
+      '_boundary_country',
+      '_categories'
+    ];
 
-module.exports.tests.interface = function(test, common) {
+    const req = {};
-  test('sanitize interface', function(t) {
+    const res = {};
-    t.equal(typeof sanitize, 'function', 'sanitize is a function');
-    t.equal(sanitize.length, 2, 'sanitize interface');
-    t.end();
-  });
-  test('middleware interface', function(t) {
-    t.equal(typeof middleware, 'function', 'middleware is a function');
-    t.equal(middleware.length, 3, 'sanitizee has a valid middleware');
-    t.end();
-  });
-};
 
-module.exports.tests.sanitizers = function(test, common) {
+    nearby.middleware(req, res, () => {
-  test('check sanitizer list', function (t) {
+      t.deepEquals(called_sanitizers, expected_sanitizers);
-    var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
-      'sources', 'sources_and_layers', 'geonames_deprecation', 'size', 'private',
-      'geo_reverse', 'boundary_country', 'categories'];
-    t.deepEqual(Object.keys(nearby.sanitizer_list), expected);
       t.end();
     });
-};
-
-module.exports.tests.middleware_success = function(test, common) {
-  test('middleware success', function(t) {
-    var req = { query: { 'point.lat': 0, 'point.lon': 0 }};
-    var next = function(){
-      t.deepEqual(req.errors, [], 'no error message set');
-      t.deepEqual(req.clean, defaultClean);
-      t.end();
-    };
-    middleware( req, undefined, next );
   });
 };
 
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /nearby ' + name, testFunction);
+    return tape('SANITIZE /nearby ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/place.js

@@ -1,111 +1,74 @@
-var place  = require('../../../sanitizer/place'),
+const  _ = require('lodash'),
-    sanitize = place.sanitize,
+    proxyquire =  require('proxyquire').noCallThru();
-    middleware = place.middleware,
-    defaultClean = { ids: [ { source: 'geonames', layer: 'venue', id: '123' } ], private: false };
 
-// these are the default values you would expect when no input params are specified.
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.sanitize = function(test, common) {
-  test('sanitize interface', function(t) {
+  test('verify that all sanitizers were called as expected', function(t) {
-    t.equal(typeof sanitize, 'function', 'sanitize is a function');
+    var called_sanitizers = [];
-    t.equal(sanitize.length, 2, 'sanitize interface');
-    t.end();
-  });
-  test('middleware interface', function(t) {
-    t.equal(typeof middleware, 'function', 'middleware is a function');
-    t.equal(middleware.length, 3, 'sanitize has a valid middleware');
-    t.end();
-  });
-};
 
-module.exports.tests.sanitizers = function(test, common) {
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
-  test('check sanitizer list', function (t) {
+    //  were all called correctly
-    var expected = ['singleScalarParameters', 'ids', 'private' ];
+    var place = proxyquire('../../../sanitizer/place.js', {
-    t.deepEqual(Object.keys(place.sanitizer_list), expected);
+      '../sanitizer/_single_scalar_parameters': function () {
-    t.end();
+        return {
-  });
+          sanitize: () => {
+            called_sanitizers.push('_single_scalar_parameters');
+            return { errors: [], warnings: [] };
+          }
         };
-
+      },
-module.exports.tests.sanitize_private = function(test, common) {
+      '../sanitizer/_debug': () => {
-  var invalid_values = [null, -1, 123, NaN, 'abc'];
+        return {
-  invalid_values.forEach(function(value) {
+          sanitize: () => {
-    test('invalid private param ' + value, function(t) {
+            called_sanitizers.push('_debug');
-      var req = { query: { ids:'geonames:venue:123', 'private': value } };
+            return { errors: [], warnings: [] };
-      sanitize(req, function(){
+          }
-        t.deepEqual( req.errors, [], 'no errors' );
+        };
-        t.deepEqual( req.warnings, [], 'no warnings' );
+      },
-        t.equal(req.clean.private, false, 'default private set (to false)');
+      '../sanitizer/_ids': function () {
-        t.end();
+        return {
-      });
+          sanitize: () => {
-    });
+            called_sanitizers.push('_ids');
-  });
+            return { errors: [], warnings: [] };
-
+          }
-  var valid_values = ['true', true, 1];
+        };
-  valid_values.forEach(function(value) {
+      },
-    test('valid private param ' + value, function(t) {
+      '../sanitizer/_flag_bool': function () {
-      var req = { query: { ids:'geonames:venue:123', 'private': value } };
+        if (arguments[0] === 'private' && arguments[1] === false) {
-      sanitize(req, function(){
+          return {
-        t.deepEqual( req.errors, [], 'no errors' );
+            sanitize: () => {
-        t.deepEqual( req.warnings, [], 'no warnings' );
+                called_sanitizers.push('_flag_bool');
-        t.equal(req.clean.private, true, 'private set to true');
+                return { errors: [], warnings: [] };
-        t.end();
+              }
-      });
+            };
-    });
+        } else {
+            throw new Error('incorrect parameters passed to _flag_bool');
+        }
+      }
     });
 
-  var valid_false_values = ['false', false, 0];
+    const expected_sanitizers = [
-  valid_false_values.forEach(function(value) {
+      '_single_scalar_parameters',
-    test('test setting false explicitly ' + value, function(t) {
+      '_debug',
-      var req = { query: { ids:'geonames:venue:123', 'private': value } };
+      '_ids',
-      sanitize(req, function(){
+      '_flag_bool'
-        t.deepEqual( req.errors, [], 'no errors' );
+    ];
-        t.deepEqual( req.warnings, [], 'no warnings' );
-        t.equal(req.clean.private, false, 'private set to false');
-        t.end();
-      });
-    });
-  });
 
-  test('test default behavior', function(t) {
+    const req = {};
-    var req = { query: { ids:'geonames:venue:123' } };
+    const res = {};
-    sanitize(req, function(){
-      t.deepEqual( req.errors, [], 'no errors' );
-      t.deepEqual( req.warnings, [], 'no warnings' );
-      t.equal(req.clean.private, false, 'private set to false');
-      t.end();
-    });
-  });
-};
 
-module.exports.tests.invalid_params = function(test, common) {
+    place.middleware(req, res, () => {
-  test('no params', function(t) {
+      t.deepEquals(called_sanitizers, expected_sanitizers);
-    var req = { query: {} };
-    sanitize( req, function(){
-      t.equal( req.errors[0], 'invalid param \'ids\': length must be >0', 'error for missing `ids` param');
-      t.deepEqual( req.warnings, [], 'no warnings' );
       t.end();
     });
   });
 };
 
-module.exports.tests.middleware_success = function(test, common) {
-  test('middleware success', function(t) {
-    var req = { query: { ids: 'geonames:venue:123' }};
-    var next = function(){
-      t.deepEqual( req.errors, [], 'no errors' );
-      t.deepEqual( req.warnings, [], 'no warnings' );
-      t.deepEqual(req.clean, defaultClean);
-      t.end();
-    };
-    middleware( req, undefined, next );
-  });
-};
-
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /place ' + name, testFunction);
+    return tape('SANITIZE /place ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/sanitizer/reverse.js

@@ -1,199 +1,137 @@
-
+const  _ = require('lodash'),
-// @todo: refactor this test, it's pretty messy, brittle and hard to follow
+    proxyquire =  require('proxyquire').noCallThru();
-
-var reverse  = require('../../../sanitizer/reverse'),
-    sanitize = reverse.sanitize,
-    middleware = reverse.middleware,
-    defaults = require('../../../query/reverse_defaults'),
-    defaultError = 'missing param \'lat\'',
-    defaultClean =  { 'point.lat': 0,
-                      'point.lon': 0,
-                      'boundary.circle.lat': 0,
-                      'boundary.circle.lon': 0,
-                      size: 10,
-                      private: false
-                    };
-
-// these are the default values you would expect when no input params are specified.
-// @todo: why is this different from $defaultClean?
-var emptyClean = { private: false, size: 10 };
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
+module.exports.tests.sanitize = function(test, common) {
-  test('sanitize interface', function(t) {
+  test('verify that all sanitizers were called as expected', function(t) {
-    t.equal(typeof sanitize, 'function', 'sanitize is a function');
+    var called_sanitizers = [];
-    t.equal(sanitize.length, 2, 'sanitize interface');
+
-    t.end();
+    // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
-  });
+    //  were all called correctly
-  test('middleware interface', function(t) {
+    var reverse = proxyquire('../../../sanitizer/reverse.js', {
-    t.equal(typeof middleware, 'function', 'middleware is a function');
+      '../sanitizer/_single_scalar_parameters': function () {
-    t.equal(middleware.length, 3, 'sanitizee has a valid middleware');
+        return {
-    t.end();
+          sanitize: () => {
-  });
+            called_sanitizers.push('_single_scalar_parameters');
+            return { errors: [], warnings: [] };
+          }
         };
-
+      },
-module.exports.tests.sanitizers = function(test, common) {
+      '../sanitizer/_debug': () => {
-  test('check sanitizer list', function (t) {
+        return {
-    var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
+          sanitize: () => {
-      'sources', 'sources_and_layers', 'geonames_deprecation', 'size', 'private',
+            called_sanitizers.push('_debug');
-      'geo_reverse', 'boundary_country'];
+            return { errors: [], warnings: [] };
-    t.deepEqual(Object.keys(reverse.sanitizer_list), expected);
+          }
-    t.end();
-  });
         };
-
+      },
-module.exports.tests.sanitize_lat = function(test, common) {
+      '../sanitizer/_deprecate_quattroshapes': function () {
-  var lats = {
+        return {
-    invalid: [],
+          sanitize: () => {
-    valid: [ 0, 45, 90, -0, '0', '45', '90', -181, -120, -91, 91, 120, 181 ],
+            called_sanitizers.push('_deprecate_quattroshapes');
-    missing: ['', undefined, null]
+            return { errors: [], warnings: [] };
+          }
         };
-  test('invalid lat', function(t) {
+      },
-    lats.invalid.forEach( function( lat ){
+      '../sanitizer/_targets': function (type) {
-      var req = { query: { 'point.lat': lat, 'point.lon': 0 } };
+        if (['layers', 'sources'].indexOf(type) !== -1) {
-      sanitize(req, function(){
+          return {
-        t.equal(req.errors[0], 'invalid param \'point.lat\': must be >-90 and <90', lat + ' is an invalid latitude');
+            sanitize: () => {
-        t.deepEqual(req.clean, emptyClean, 'clean only has default values set');
+              called_sanitizers.push(`_targets/${type}`);
-      });
+              return { errors: [], warnings: [] };
-    });
+              }
-    t.end();
-  });
-  test('valid lat', function(t) {
-    lats.valid.forEach( function( lat ){
-      var req = { query: { 'point.lat': lat, 'point.lon': 0 } };
-      sanitize(req, function(){
-        var expected_lat = parseFloat( lat );
-        t.deepEqual(req.errors, [], 'no errors');
-      });
-    });
-    t.end();
-  });
-  test('missing lat', function(t) {
-    lats.missing.forEach( function( lat ){
-      var req = { query: { 'point.lat': lat, 'point.lon': 0 } };
-      sanitize(req, function(){
-        t.equal(req.errors[0], 'missing param \'point.lat\'', 'latitude is a required field');
-        t.deepEqual(req.clean, emptyClean, 'clean only has default values set');
-      });
-    });
-    t.end();
-  });
             };
-
+          } else {
-module.exports.tests.sanitize_lon = function(test, common) {
+          throw new Error('incorrect parameters passed to _targets');
-  var lons = {
+        }
-    valid: [ -360, -181, 181, -180, -1, -0, 0, 45, 90, '-180', '0', '180' ],
+      },
-    missing: ['', undefined, null]
+      '../sanitizer/_sources_and_layers': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_sources_and_layers');
+            return { errors: [], warnings: [] };
+          }
         };
-  test('valid lon', function(t) {
+      },
-    lons.valid.forEach( function( lon ){
+      '../sanitizer/_geonames_deprecation': function () {
-      var req = { query: { 'point.lat': 0, 'point.lon': lon } };
+        return {
-      sanitize(req, function(){
+          sanitize: () => {
-        var expected_lon = parseFloat( lon );
+            called_sanitizers.push('_geonames_deprecations');
-        t.deepEqual(req.errors, [], 'no errors');
+            return { errors: [], warnings: [] };
-      });
+          }
-    });
-    t.end();
-  });
-  test('missing lon', function(t) {
-    lons.missing.forEach( function( lon ){
-      var req = { query: { 'point.lat': 0, 'point.lon': lon } };
-
-      // @todo: why is lat set?
-      var expected = { 'point.lat': 0, private: false, size: 10 };
-      sanitize(req, function(){
-        t.equal(req.errors[0], 'missing param \'point.lon\'', 'longitude is a required field');
-        t.deepEqual(req.clean, expected, 'clean only has default values set');
-      });
-    });
-    t.end();
-  });
         };
-
+      },
-module.exports.tests.sanitize_size = function(test, common) {
+      '../sanitizer/_size': function () {
-  test('invalid size value', function(t) {
+        if (_.isEmpty(arguments)) {
-    var req = { query: { size: 'a', 'point.lat': 0, 'point.lon': 0 } };
+          return {
-    sanitize(req, function(){
+            sanitize: () => {
-      t.equal(req.clean.size, 10, 'default size set');
+              called_sanitizers.push('_size');
-      t.end();
+              return { errors: [], warnings: [] };
-    });
+              }
-  });
-  test('below min size value', function(t) {
-    var req = { query: { size: -100, 'point.lat': 0, 'point.lon': 0 } };
-    sanitize(req, function(){
-      t.equal(req.clean.size, 1, 'min size set');
-      t.end();
-    });
-  });
-  test('above max size value', function(t) {
-    var req = { query: { size: 9999, 'point.lat': 0, 'point.lon': 0 } };
-    sanitize(req, function(){
-      t.equal(req.clean.size, 40, 'max size set');
-      t.end();
-    });
-  });
           };
-
+        } else {
-module.exports.tests.sanitize_private = function(test, common) {
+          throw new Error('should not have passed any parameters to _size');
-  var invalid_values = [null, -1, 123, NaN, 'abc'];
+        }
-  invalid_values.forEach(function(value) {
+      },
-    test('invalid private param ' + value, function(t) {
+      '../sanitizer/_flag_bool': function () {
-      var req = { query: { 'point.lat': 0, 'point.lon': 0, 'private': value } };
+        if (arguments[0] === 'private' && arguments[1] === false) {
-      sanitize(req, function(){
+          return {
-        t.equal(req.clean.private, false, 'default private set (to false)');
+            sanitize: () => {
-        t.end();
+                called_sanitizers.push('_flag_bool');
-      });
+                return { errors: [], warnings: [] };
-    });
+              }
+            };
+        } else {
+            throw new Error('incorrect parameters passed to _flag_bool');
+        }
+      },
+      '../sanitizer/_geo_reverse': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_geo_reverse');
+            return { errors: [], warnings: [] };
+          }
+        };
+      },
+      '../sanitizer/_boundary_country': function () {
+        return {
+          sanitize: () => {
+            called_sanitizers.push('_boundary_country');
+            return { errors: [], warnings: [] };
+          }
+        };
+      }
     });
 
-  var valid_values = ['true', true, 1, '1'];
+    const expected_sanitizers = [
-  valid_values.forEach(function(value) {
+      '_single_scalar_parameters',
-    test('valid private param ' + value, function(t) {
+      '_debug',
-      var req = { query: { 'point.lat': 0, 'point.lon': 0, 'private': value } };
+      '_deprecate_quattroshapes',
-      sanitize(req, function(){
+      '_targets/layers',
-        t.equal(req.clean.private, true, 'private set to true');
+      '_targets/sources',
-        t.end();
+      '_sources_and_layers',
-      });
+      '_geonames_deprecations',
-    });
+      '_size',
-  });
+      '_flag_bool',
+      '_geo_reverse',
+      '_boundary_country'
+    ];
 
-  var valid_false_values = ['false', false, 0];
+    const req = {};
-  valid_false_values.forEach(function(value) {
+    const res = {};
-    test('test setting false explicitly ' + value, function(t) {
-      var req = { query: { 'point.lat': 0, 'point.lon': 0, 'private': value } };
-      sanitize(req, function(){
-        t.equal(req.clean.private, false, 'private set to false');
-        t.end();
-      });
-    });
-  });
 
-  test('test default behavior', function(t) {
+    reverse.middleware(req, res, () => {
-    var req = { query: { 'point.lat': 0, 'point.lon': 0 } };
+      t.deepEquals(called_sanitizers, expected_sanitizers);
-    sanitize(req, function(){
-      t.equal(req.clean.private, false, 'private set to false');
       t.end();
     });
   });
 };
 
-module.exports.tests.middleware_success = function(test, common) {
-  test('middleware success', function(t) {
-    var req = { query: { 'point.lat': 0, 'point.lon': 0 }};
-    var next = function(){
-      t.deepEqual(req.errors, [], 'no error message set');
-      t.deepEqual(req.clean, defaultClean);
-      t.end();
-    };
-    middleware( req, undefined, next );
-  });
-};
-
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /reverse ' + name, testFunction);
+    return tape('SANITIZE /reverse ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){