Merge pull request #916 from pelias/staging

Merge staging into production

.lgtm

@@ -1 +0,0 @@
-pattern = "(?i):shipit:|:\\+1:|LGTM"

README.md

@@ -44,10 +44,8 @@ The API recognizes the following properties under the top-level `api` key in you
 |`indexName`|*no*|*pelias*|name of the Elasticsearch index to be used when building queries|
 |`legacyUrl`|*no*||the url to redirect to in case the user does not specify a version such as `v1`
 |`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable
-|`accessLog`|*no*||name of the format to use for access logs; may be any one of the
+|`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.|
-  [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to
+|`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service) and [placholder](https://github.com/pelias/placeholder) services.  If missing (which is not recommended), the point-in-polygon and placeholder services will not be called.|
-  `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.|
-|`pipService`|*yes*||full url to the pip service to be used for coarse reverse queries. if missing, which is not recommended, the service will default to using nearby lookups instead of point-in-polygon.|
 
 Example configuration file would look something like this:
 
@@ -70,7 +68,14 @@ Example configuration file would look something like this:
     "legacyUrl": "pelias.mapzen.com",
     "relativeScores": true,
     "textAnalyzer": "libpostal",
-    "pipService": "http://mypipservice.com/3000"
+    "services": {
+      "pip": {
+        "url": "http://mypipservice.com:3000"
+      },
+      "placeholder": {
+        "url": "http://myplaceholderservice.com:5000"
+      }
+    }
   },
   "interpolation": {
     "client": {

controller/coarse_reverse.js

@@ -1,9 +1,9 @@
 const logger = require('pelias-logger').get('coarse_reverse');
 const _ = require('lodash');
 const Document = require('pelias-model').Document;
-const isDNT = require('../helper/logging').isDNT;
 
-const granularities = [
+// do not change order, other functionality depends on most-to-least granular order
+const coarse_granularities = [
   'neighbourhood',
   'borough',
   'locality',
@@ -16,60 +16,79 @@ const granularities = [
   'country'
 ];
 
-function getMostGranularLayer(results) {
+// remove non-coarse layers and return what's left (or all if empty)
-  return granularities.find((granularity) => {
+function getEffectiveLayers(requested_layers) {
-    return results.hasOwnProperty(granularity);
+  // remove non-coarse layers
+  const non_coarse_layers_removed = _.without(requested_layers, 'venue', 'address', 'street');
+
+  // if resulting array is empty, use all coarse granularities
+  if (_.isEmpty(non_coarse_layers_removed)) {
+    return coarse_granularities;
+  }
+
+  // otherwise use requested layers with non-coarse layers removed
+  return non_coarse_layers_removed;
+
+}
+
+// drop from coarse_granularities until there's one that was requested
+// this depends on coarse_granularities being ordered
+function getApplicableRequestedLayers(requested_layers) {
+  return _.dropWhile(coarse_granularities, (coarse_granularity) => {
+    return !_.includes(requested_layers, coarse_granularity);
   });
 }
 
-function hasResultsAtRequestedLayers(results, layers) {
+//  removing non-coarse layers could leave effective_layers empty, so it's
-  return _.intersection(layers, Object.keys(results)).length > 0;
+//  important to check for empty layers here
+function hasResultsAtRequestedLayers(results, requested_layers) {
+  return !_.isEmpty(_.intersection(_.keys(results), requested_layers));
+}
+
+// get the most granular layer from the results by taking the head of the intersection
+// of coarse_granularities (which are ordered) and the result layers
+// ['neighbourhood', 'borough', 'locality'] - ['locality', 'borough'] = 'borough'
+// this depends on coarse_granularities being ordered
+function getMostGranularLayerOfResult(result_layers) {
+  return _.head(_.intersection(coarse_granularities, result_layers));
 }
 
+// create a model.Document from what's left, using the most granular
+// result available as the starting point
 function synthesizeDoc(results) {
-  // now create a model.Document from what's level, using the most granular
+  // find the most granular layer to use as the document layer
-  // result available as the starting point
+  const most_granular_layer = getMostGranularLayerOfResult(_.keys(results));
-  // the immediately above cannot be re-used since county may be the most
-  // granular layer requested but the results may start at region (no county found)
-  const most_granular_layer = getMostGranularLayer(results);
   const id = results[most_granular_layer][0].id;
 
   const doc = new Document('whosonfirst', most_granular_layer, id.toString());
   doc.setName('default', results[most_granular_layer][0].name);
 
-  if (results[most_granular_layer][0].hasOwnProperty('centroid')) {
+  // assign the administrative hierarchy
+  _.keys(results).forEach((layer) => {
+    doc.addParent(layer, results[layer][0].name, results[layer][0].id.toString(), results[layer][0].abbr);
+  });
+
+  // set centroid if available
+  if (_.has(results[most_granular_layer][0], 'centroid')) {
     doc.setCentroid( results[most_granular_layer][0].centroid );
   }
 
-  if (results[most_granular_layer][0].hasOwnProperty('bounding_box')) {
+  // set bounding box if available
-    const parsedBoundingBox = results[most_granular_layer][0].bounding_box.split(',').map(parseFloat);
+  if (_.has(results[most_granular_layer][0], 'bounding_box')) {
+    const parsed_bounding_box = results[most_granular_layer][0].bounding_box.split(',').map(parseFloat);
     doc.setBoundingBox({
       upperLeft: {
-        lat: parsedBoundingBox[3],
+        lat: parsed_bounding_box[3],
-        lon: parsedBoundingBox[0]
+        lon: parsed_bounding_box[0]
       },
       lowerRight: {
-        lat: parsedBoundingBox[1],
+        lat: parsed_bounding_box[1],
-        lon: parsedBoundingBox[2]
+        lon: parsed_bounding_box[2]
       }
     });
 
   }
 
-  if (_.has(results, 'country[0].abbr')) {
-    doc.setAlpha3(results.country[0].abbr);
-  }
-
-  // assign the administrative hierarchy
-  Object.keys(results).forEach((layer) => {
-    if (results[layer][0].hasOwnProperty('abbr')) {
-      doc.addParent(layer, results[layer][0].name, results[layer][0].id.toString(), results[layer][0].abbr);
-    } else {
-      doc.addParent(layer, results[layer][0].name, results[layer][0].id.toString());
-    }
-
-  });
-
   const esDoc = doc.toESDocument();
   esDoc.data._id = esDoc._id;
   esDoc.data._type = esDoc._type;
@@ -84,12 +103,21 @@ function setup(service, should_execute) {
       return next();
     }
 
+    // return a warning to the caller that boundary.circle.radius will be ignored
+    if (!_.isUndefined(req.clean['boundary.circle.radius'])) {
+      req.warnings.push('boundary.circle.radius is not applicable for coarse reverse');
+    }
+
+    // because coarse reverse is called when non-coarse reverse didn't return
+    //  anything, treat requested layers as if it didn't contain non-coarse layers
+    const effective_layers = getEffectiveLayers(req.clean.layers);
+
     const centroid = {
       lat: req.clean['point.lat'],
       lon: req.clean['point.lon']
     };
 
-    service(centroid, isDNT(req), (err, results) => {
+    service(req, (err, results) => {
       // if there's an error, log it and bail
       if (err) {
         logger.info(`[controller:coarse_reverse][error]`);
@@ -97,27 +125,20 @@ function setup(service, should_execute) {
         return next();
       }
 
-      // find the finest granularity requested
+      // log how many results there were
-      const finest_granularity_requested = granularities.findIndex((granularity) => {
+      logger.info(`[controller:coarse_reverse][queryType:pip][result_count:${_.size(results)}]`);
-        return req.clean.layers.indexOf(granularity) !== -1;
-      });
 
-      logger.info(`[controller:coarse_reverse][queryType:pip][result_count:${Object.keys(results).length}]`);
+      // now keep everything from the response that is equal to or less granular
-
+      // than the most granular layer requested.  that is, if effective_layers=['county'],
-      // now remove everything from the response that is more granular than the
+      // remove neighbourhoods, boroughs, localities, localadmins
-      // most granular layer requested.  that is, if req.clean.layers=['county'],
+      const applicable_results = _.pick(results, getApplicableRequestedLayers(effective_layers));
-      // remove neighbourhoods, localities, and localadmins
-      Object.keys(results).forEach((layer) => {
-        if (granularities.indexOf(layer) < finest_granularity_requested) {
-          delete results[layer];
-        }
-      });
 
       res.meta = {};
       res.data = [];
-      // synthesize a doc from results if there's a result at the request layer(s)
+
-      if (hasResultsAtRequestedLayers(results, req.clean.layers)) {
+      // if there's a result at the requested layer(s), synthesize a doc from results
-        res.data.push(synthesizeDoc(results));
+      if (hasResultsAtRequestedLayers(applicable_results, effective_layers)) {
+        res.data.push(synthesizeDoc(applicable_results));
       }
 
       return next();

controller/placeholder.js

@@ -0,0 +1,271 @@
+const _ = require('lodash');
+const logger = require('pelias-logger').get('api');
+const Document = require('pelias-model').Document;
+const geolib = require('geolib');
+
+// composition of toNumber and isFinite, useful for single call to convert a value
+//  to a number, then checking to see if it's finite
+function isFiniteNumber(value) {
+  return !_.isEmpty(_.trim(value)) && _.isFinite(_.toNumber(value));
+}
+
+// returns true if value is parseable as finite non-negative number
+function isNonNegativeFiniteNumber(value) {
+  return isFiniteNumber(value) && _.gte(value, 0);
+}
+
+function hasLatLon(result) {
+  return _.isFinite(_.get(result.geom, 'lat')) && _.isFinite(_.get(result.geom, 'lon'));
+}
+
+function getLatLon(result) {
+  return {
+    latitude: result.geom.lat,
+    longitude: result.geom.lon
+  };
+}
+
+// if geom.lat/lon are parseable as finite numbers, convert to a finite number
+// otherwise remove the field
+function numberifyGeomLatLon(result) {
+  ['lat', 'lon'].forEach((f) => {
+    if (isFiniteNumber(_.get(result.geom, f))) {
+      result.geom[f] = _.toFinite(result.geom[f]);
+    } else {
+      // result.geom may not exist, so use unset instead of delete
+      _.unset(result.geom, f);
+    }
+  });
+
+  return result;
+
+}
+
+// returns true if all 4 ,-delimited (max) substrings are parseable as finite numbers
+// '12.12,21.21,13.13,31.31'       returns true
+// '12.12,21.21,13.13,31.31,14.14' returns false
+// '12.12,21.21,13.13,blah'        returns false
+// '12.12,21.21,13.13,31.31,blah'  returns false
+// '12.12,NaN,13.13,31.31'         returns false
+// '12.12,Infinity,13.13,31.31'    returns false
+function is4CommaDelimitedNumbers(bbox) {
+  return _.defaultTo(bbox, '').
+    split(',').
+    filter(isFiniteNumber).length === 4;
+}
+
+function hasName(result) {
+  return !_.isEmpty(_.trim(result.name));
+}
+
+// filter that passes only results that match on requested layers
+// passes everything if req.clean.layers is not found
+function getLayersFilter(clean) {
+  if (_.isEmpty(_.get(clean, 'layers', []))) {
+    return _.constant(true);
+  }
+
+  // otherwise return a function that checks for set inclusion of a result placetype
+  return (result) => {
+    return _.includes(clean.layers, result.placetype);
+  };
+
+}
+
+// return true if the hierarchy does not have a country.abbr
+// OR hierarchy country.abbr matches boundary.country
+function matchesBoundaryCountry(boundaryCountry, hierarchy) {
+  return !boundaryCountry || _.get(hierarchy, 'country.abbr') === boundaryCountry;
+}
+
+// return true if the result does not have a lineage
+// OR at least one lineage matches the requested boundary.country
+function atLeastOneLineageMatchesBoundaryCountry(boundaryCountry, result) {
+  return !result.lineage || result.lineage.some(_.partial(matchesBoundaryCountry, boundaryCountry));
+}
+
+// return a function that detects if a result has at least one lineage in boundary.country
+// if there's no boundary.country, return a function that always returns true
+function getBoundaryCountryFilter(clean) {
+  if (_.has(clean, 'boundary.country')) {
+    return _.partial(atLeastOneLineageMatchesBoundaryCountry, clean['boundary.country']);
+  }
+
+  return _.constant(true);
+
+}
+
+// return a function that detects if a result is inside a bbox if a bbox is available
+// if there's no bbox, return a function that always returns true
+function getBoundaryRectangleFilter(clean) {
+  if (['min_lat', 'min_lon', 'max_lat', 'max_lon'].every((f) => {
+    return _.has(clean, `boundary.rect.${f}`);
+  })) {
+    const polygon = [
+      { latitude: clean['boundary.rect.min_lat'], longitude: clean['boundary.rect.min_lon'] },
+      { latitude: clean['boundary.rect.max_lat'], longitude: clean['boundary.rect.min_lon'] },
+      { latitude: clean['boundary.rect.max_lat'], longitude: clean['boundary.rect.max_lon'] },
+      { latitude: clean['boundary.rect.min_lat'], longitude: clean['boundary.rect.max_lon'] }
+    ];
+    const isPointInsidePolygon = _.partialRight(geolib.isPointInside, polygon);
+
+    return _.partial(isInsideGeometry, isPointInsidePolygon);
+
+  }
+
+  return _.constant(true);
+
+}
+
+// return a function that detects if a result is inside a circle if a circle is available
+// if there's no circle, return a function that always returns true
+function getBoundaryCircleFilter(clean) {
+  if (['lat', 'lon', 'radius'].every((f) => {
+    return _.has(clean, `boundary.circle.${f}`);
+  })) {
+    const center = {
+      latitude: clean['boundary.circle.lat'],
+      longitude: clean['boundary.circle.lon']
+    };
+    const radiusInMeters = clean['boundary.circle.radius'] * 1000;
+    const isPointInCircle = _.partialRight(geolib.isPointInCircle, center, radiusInMeters);
+
+    return _.partial(isInsideGeometry, isPointInCircle);
+
+  }
+
+  return _.constant(true);
+
+}
+
+// helper that calls an "is inside some geometry" function
+function isInsideGeometry(f, result) {
+  return hasLatLon(result) ? f(getLatLon(result)) : false;
+}
+
+function placetypeHasNameAndId(hierarchyElement) {
+  return !_.isEmpty(_.trim(hierarchyElement.name)) &&
+          !_.isEmpty(_.trim(hierarchyElement.id));
+}
+
+// synthesize an ES doc from a placeholder result
+function synthesizeDocs(boundaryCountry, result) {
+  const doc = new Document('whosonfirst', result.placetype, result.id.toString());
+  doc.setName('default', result.name);
+
+  // only assign centroid if both lat and lon are finite numbers
+  if (hasLatLon(result)) {
+    doc.setCentroid( { lat: result.geom.lat, lon: result.geom.lon } );
+  } else {
+    logger.error(`could not parse centroid for id ${result.id}`);
+  }
+
+  // lodash conformsTo verifies that an object has a property with a certain format
+  if (_.conformsTo(result.geom, { 'bbox': is4CommaDelimitedNumbers } )) {
+    const parsedBoundingBox = result.geom.bbox.split(',').map(_.toFinite);
+    doc.setBoundingBox({
+      upperLeft: {
+        lat: parsedBoundingBox[3],
+        lon: parsedBoundingBox[0]
+      },
+      lowerRight: {
+        lat: parsedBoundingBox[1],
+        lon: parsedBoundingBox[2]
+      }
+    });
+  } else {
+    logger.error(`could not parse bbox for id ${result.id}: ${_.get(result, 'geom.bbox')}`);
+  }
+
+  // set population and popularity if parseable as finite number
+  if (isNonNegativeFiniteNumber(result.population)) {
+    doc.setPopulation(_.toFinite(result.population));
+  }
+  if (isNonNegativeFiniteNumber(result.popularity)) {
+    doc.setPopularity(_.toFinite(result.popularity));
+  }
+
+  _.defaultTo(result.lineage, [])
+    // remove all lineages that don't match an explicit boundary.country
+    .filter(_.partial(matchesBoundaryCountry, boundaryCountry))
+    // add all the lineages to the doc
+    .map((hierarchy) => {
+      Object.keys(hierarchy)
+        .filter(doc.isSupportedParent)
+        .filter((placetype) => {
+          return placetypeHasNameAndId(hierarchy[placetype]);
+        })
+        .forEach((placetype) => {
+          doc.addParent(
+            placetype,
+            hierarchy[placetype].name,
+            hierarchy[placetype].id.toString(),
+            hierarchy[placetype].abbr);
+      });
+    });
+
+  return buildESDoc(doc);
+
+}
+
+function buildESDoc(doc) {
+  const esDoc = doc.toESDocument();
+  return _.extend(esDoc.data, { _id: esDoc._id, _type: esDoc._type });
+}
+
+function setup(placeholderService, should_execute) {
+  function controller( req, res, next ){
+    // bail early if req/res don't pass conditions for execution
+    if (!should_execute(req, res)) {
+      return next();
+    }
+
+    placeholderService(req, (err, results) => {
+      if (err) {
+        // bubble up an error if one occurred
+        if (_.isObject(err) && err.message) {
+          req.errors.push( err.message );
+        } else {
+          req.errors.push( err );
+        }
+
+      } else {
+        const boundaryCountry = _.get(req, ['clean', 'boundary.country']);
+
+        // convert results to ES docs
+        // boundary.country filter must happen after synthesis since multiple
+        //  lineages may produce different country docs
+        res.meta = {};
+        res.data = results
+                    // filter out results that don't have a name
+                    .filter(hasName)
+                    // filter out results that don't match on requested layer(s)
+                    .filter(getLayersFilter(req.clean))
+                    // filter out results that don't match on any lineage country
+                    .filter(getBoundaryCountryFilter(req.clean))
+                    // clean up geom.lat/lon for boundary rect/circle checks
+                    .map(numberifyGeomLatLon)
+                    // filter out results that aren't in the boundary.rect
+                    .filter(getBoundaryRectangleFilter(req.clean))
+                    // filter out results that aren't in the boundary.circle
+                    .filter(getBoundaryCircleFilter(req.clean))
+                    // convert results to ES docs
+                    .map(_.partial(synthesizeDocs, boundaryCountry));
+
+        const messageParts = [
+          '[controller:placeholder]',
+          `[result_count:${_.defaultTo(res.data, []).length}]`
+        ];
+
+        logger.info(messageParts.join(' '));
+      }
+
+      return next();
+    });
+
+  }
+
+  return controller;
+}
+
+module.exports = setup;

controller/predicates/has_results_at_layers.js

@@ -0,0 +1,19 @@
+const _ = require('lodash');
+
+// returns a function that returns true if any result.layer is in any of the
+// supplied layers using array intersection
+
+// example usage: determining if the response contains only admin results
+
+module.exports = (layers) => {
+  return (request, response) => {
+    return !_.isEmpty(
+      _.intersection(
+        // convert layers to an array if it isn't already one
+        _.castArray(layers),
+        // pull all the layer properties into an array
+        _.map(response.data, _.property('layer'))
+    ));
+  };
+
+};

controller/predicates/is_admin_only_analysis.js

@@ -0,0 +1,13 @@
+const _ = require('lodash');
+
+module.exports = (request, response) => {
+  if (!request.clean.hasOwnProperty('parsed_text')) {
+    return false;
+  }
+
+  // return true only if all non-admin properties of parsed_text are empty
+  return ['number', 'street', 'query', 'category'].every((prop) => {
+    return _.isEmpty(request.clean.parsed_text[prop]);
+  });
+
+};

controller/predicates/is_coarse_reverse.js

@@ -5,5 +5,5 @@ const non_coarse_layers = ['address', 'street', 'venue'];
 module.exports = (req, res) => {
   // returns true if layers is undefined, empty, or contains 'address', 'street', or 'venue'
   return !_.isEmpty(req.clean.layers) &&
-          _.intersection(req.clean.layers, non_coarse_layers).length === 0;
+          _.isEmpty(_.intersection(req.clean.layers, non_coarse_layers));
 };

controller/predicates/is_pip_service_enabled.js

@@ -1,7 +0,0 @@
-module.exports = (uri) => {
-  // this predicate relies upon the fact that the schema has already validated
-  //  that api.pipService is a URI-formatted string
-  return (request, response) => {
-    return uri !== undefined;
-  };
-};

controller/predicates/is_service_enabled.js

@@ -0,0 +1,5 @@
+module.exports = (uri) => {
+  return (request, response) => {
+    return uri !== undefined;
+  };
+};

helper/geojsonify.js

@@ -1,6 +1,6 @@
 
 var GeoJSON = require('geojson');
-var extent = require('geojson-extent');
+var extent = require('@mapbox/geojson-extent');
 var logger = require('pelias-logger').get('api');
 var type_mapping = require('./type_mapping');
 var _ = require('lodash');

middleware/normalizeParentIds.js

@@ -1,7 +1,9 @@
-var logger = require('pelias-logger').get('api');
+'use strict';
-var Document = require('pelias-model').Document;
 
-var placeTypes = require('../helper/placeTypes');
+const logger = require('pelias-logger').get('api');
+const Document = require('pelias-model').Document;
+const placeTypes = require('../helper/placeTypes');
+const _ = require('lodash');
 
 /**
  * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint.
@@ -32,16 +34,18 @@ function normalizeParentIds(place) {
   if (place) {
     placeTypes.forEach(function (placeType) {
       if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) {
-        var source = 'whosonfirst';
+        let source = 'whosonfirst';
+
+        const placetype_ids = _.get(place, `${placeType}_gid`, [null]);
 
         // looking forward to the day we can remove all geonames specific hacks, but until then...
         // geonames sometimes has its own ids in the parent hierarchy, so it's dangerous to assume that 
         // it's always WOF ids and hardcode to that
-        if (place.source === 'geonames' && place.source_id === place[placeType + '_gid'][0]) {
+        if (place.source === 'geonames' && place.source_id === placetype_ids[0]) {
           source = place.source;
         }
         
-        place[placeType + '_gid'] = [ makeNewId(source, placeType, place[placeType + '_gid']) ];
+        place[`${placeType}_gid`] = [ makeNewId(source, placeType, placetype_ids[0]) ];
       }
     });
   }

middleware/sortResponseData.js

@@ -0,0 +1,35 @@
+const _ = require('lodash');
+
+const logger = require('pelias-logger').get('api');
+
+function setup(comparator, should_execute) {
+  function middleware(req, res, next) {
+    // bail early if req/res don't pass conditions for execution or there's no data to sort
+    if (!should_execute(req, res) || _.isEmpty(res.data)) {
+      return next();
+    }
+
+    // capture the pre-sort order
+    const presort_order = res.data.map(_.property('_id'));
+
+    // sort operates on array in place
+    res.data.sort(comparator(req.clean));
+
+    // capture the post-sort order
+    const postsort_order = res.data.map(_.property('_id'));
+
+    // log it for debugging purposes
+    logger.debug([
+      `req.clean: ${JSON.stringify(req.clean)}`,
+      `pre-sort: [${presort_order}]`,
+      `post-sort: [${postsort_order}]`
+    ].join(', '));
+
+    next();
+  }
+
+  return middleware;
+
+}
+
+module.exports = setup;

package.json

@@ -46,26 +46,26 @@
     "express-http-proxy": "^0.11.0",
     "extend": "^3.0.1",
     "geojson": "^0.4.0",
-    "geojson-extent": "^0.3.1",
+    "@mapbox/geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "iso-639-3": "^1.0.0",
     "iso3166-1": "^0.3.0",
-    "joi": "^10.1.0",
+    "joi": "^10.5.2",
     "locale": "^0.1.0",
-    "lodash": "^4.5.0",
+    "lodash": "^4.17.4",
     "markdown": "0.5.0",
-    "morgan": "1.8.1",
+    "morgan": "^1.8.2",
-    "pelias-config": "2.10.0",
     "pelias-categories": "1.2.0",
+    "pelias-config": "2.11.0",
     "pelias-labels": "1.6.0",
     "pelias-logger": "0.2.0",
-    "pelias-mock-logger": "^1.0.1",
+    "pelias-microservice-wrapper": "1.1.2",
     "pelias-model": "4.8.1",
     "pelias-query": "8.15.0",
-    "pelias-text-analyzer": "1.8.2",
+    "pelias-sorting": "1.0.1",
+    "pelias-text-analyzer": "1.8.3",
     "predicates": "^1.0.1",
     "retry": "^0.10.1",
-    "request": "^2.79.0",
     "stats-lite": "^2.0.4",
     "superagent": "^3.2.1",
     "through2": "^2.0.3"
@@ -77,6 +77,7 @@
     "jshint": "^2.5.6",
     "npm-check": "git://github.com/orangejulius/npm-check.git#disable-update-check",
     "nsp": "^2.2.0",
+    "pelias-mock-logger": "1.1.0",
     "precommit-hook": "^3.0.0",
     "proxyquire": "^1.7.10",
     "semantic-release": "^6.3.2",

query/reverse.js

@@ -3,6 +3,7 @@
 const peliasQuery = require('pelias-query');
 const defaults = require('./reverse_defaults');
 const check = require('check-types');
+const _ = require('lodash');
 const logger = require('pelias-logger').get('api');
 
 //------------------------------
@@ -44,7 +45,8 @@ function generateQuery( clean ){
 
   // layers
   if( check.array(clean.layers) && clean.layers.length ) {
-    vs.var( 'layers', clean.layers);
+    // only include non-coarse layers
+    vs.var( 'layers', _.intersection(clean.layers, ['address', 'street', 'venue']));
     logStr += '[param:layers] ';
   }
 

query/reverse_defaults.js

@@ -6,6 +6,7 @@ module.exports = _.merge({}, peliasQuery.defaults, {
 
   'size': 1,
   'track_scores': true,
+  'layers': ['venue', 'address', 'street'],
 
   'centroid:field': 'center_point',
 
@@ -13,7 +14,6 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'sort:distance:distance_type': 'plane',
 
   'boundary:circle:radius': '1km',
-  'boundary:circle:radius:coarse': '500km',
   'boundary:circle:distance_type': 'plane',
   'boundary:circle:optimize_bbox': 'indexed',
 

routes/v1.js

@@ -4,6 +4,7 @@ var elasticsearch = require('elasticsearch');
 const all = require('predicates').all;
 const any = require('predicates').any;
 const not = require('predicates').not;
+const _ = require('lodash');
 
 /** ----------------------- sanitizers ----------------------- **/
 var sanitizers = {
@@ -28,6 +29,7 @@ var controllers = {
   coarse_reverse: require('../controller/coarse_reverse'),
   mdToHTML: require('../controller/markdownToHtml'),
   place: require('../controller/place'),
+  placeholder: require('../controller/placeholder'),
   search: require('../controller/search'),
   status: require('../controller/status')
 };
@@ -59,16 +61,24 @@ var postProc = {
   parseBoundingBox: require('../middleware/parseBBox'),
   normalizeParentIds: require('../middleware/normalizeParentIds'),
   assignLabels: require('../middleware/assignLabels'),
-  changeLanguage: require('../middleware/changeLanguage')
+  changeLanguage: require('../middleware/changeLanguage'),
+  sortResponseData: require('../middleware/sortResponseData')
 };
 
 // predicates that drive whether controller/search runs
 const hasResponseData = require('../controller/predicates/has_response_data');
 const hasRequestErrors = require('../controller/predicates/has_request_errors');
 const isCoarseReverse = require('../controller/predicates/is_coarse_reverse');
+const isAdminOnlyAnalysis = require('../controller/predicates/is_admin_only_analysis');
+const hasResultsAtLayers = require('../controller/predicates/has_results_at_layers');
 
 // shorthand for standard early-exit conditions
 const hasResponseDataOrRequestErrors = any(hasResponseData, hasRequestErrors);
+const hasAdminOnlyResults = not(hasResultsAtLayers(['venue', 'address', 'street']));
+
+const serviceWrapper = require('pelias-microservice-wrapper').service;
+const PlaceHolder = require('../service/configurations/PlaceHolder');
+const PointInPolygon = require('../service/configurations/PointInPolygon');
 
 /**
  * Append routes to app
@@ -79,17 +89,27 @@ const hasResponseDataOrRequestErrors = any(hasResponseData, hasRequestErrors);
 function addRoutes(app, peliasConfig) {
   const esclient = elasticsearch.Client(peliasConfig.esclient);
 
-  const isPipServiceEnabled = require('../controller/predicates/is_pip_service_enabled')(peliasConfig.api.pipService);
+  const pipConfiguration = new PointInPolygon(_.defaultTo(peliasConfig.api.services.pip, {}));
-  const pipService = require('../service/pointinpolygon')(peliasConfig.api.pipService);
+  const pipService = serviceWrapper(pipConfiguration);
+  const isPipServiceEnabled = _.constant(pipConfiguration.isEnabled());
+
+  const placeholderConfiguration = new PlaceHolder(_.defaultTo(peliasConfig.api.services.placeholder, {}));
+  const placeholderService = serviceWrapper(placeholderConfiguration);
+  const isPlaceholderServiceEnabled = _.constant(placeholderConfiguration.isEnabled());
+
+  // fallback to coarse reverse when regular reverse didn't return anything
+  const coarseReverseShouldExecute = all(
+    isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData)
+  );
 
-  const coarse_reverse_should_execute = all(
+  const placeholderShouldExecute = all(
-    not(hasRequestErrors), isPipServiceEnabled, isCoarseReverse
+    not(hasResponseDataOrRequestErrors), isPlaceholderServiceEnabled, isAdminOnlyAnalysis
   );
 
   // execute under the following conditions:
   // - there are no errors or data
   // - request is not coarse OR pip service is disabled
-  const original_reverse_should_execute = all(
+  const nonCoarseReverseShouldExecute = all(
     not(hasResponseDataOrRequestErrors),
     any(
       not(isCoarseReverse),
@@ -112,6 +132,7 @@ function addRoutes(app, peliasConfig) {
       sanitizers.search.middleware,
       middleware.requestLanguage,
       middleware.calcSize(),
+      controllers.placeholder(placeholderService, placeholderShouldExecute),
       // 3rd parameter is which query module to use, use fallback/geodisambiguation
       //  first, then use original search strategy if first query didn't return anything
       controllers.search(peliasConfig.api, esclient, queries.libpostal, not(hasResponseDataOrRequestErrors)),
@@ -122,6 +143,7 @@ function addRoutes(app, peliasConfig) {
       postProc.confidenceScores(peliasConfig.api),
       postProc.confidenceScoresFallback(),
       postProc.interpolate(),
+      postProc.sortResponseData(require('pelias-sorting'), hasAdminOnlyResults),
       postProc.dedupe(),
       postProc.accuracy(),
       postProc.localNamingConventions(),
@@ -175,8 +197,8 @@ function addRoutes(app, peliasConfig) {
       sanitizers.reverse.middleware,
       middleware.requestLanguage,
       middleware.calcSize(),
-      controllers.coarse_reverse(pipService, coarse_reverse_should_execute),
+      controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute),
-      controllers.search(peliasConfig.api, esclient, queries.reverse, original_reverse_should_execute),
+      controllers.coarse_reverse(pipService, coarseReverseShouldExecute),
       postProc.distances('point.'),
       // reverse confidence scoring depends on distance from origin
       //  so it must be calculated first

sanitizer/_geo_reverse.js

@@ -4,6 +4,8 @@ var defaults = require('../query/reverse_defaults');
 var LAT_LON_IS_REQUIRED = true,
     CIRCLE_IS_REQUIRED = false;
 
+const non_coarse_layers = ['venue', 'address', 'street'];
+
 // validate inputs, convert types and apply defaults
 module.exports = function sanitize( raw, clean ){
 
@@ -29,13 +31,8 @@ module.exports = function sanitize( raw, clean ){
 
     // if no radius was passed, set the default
     if ( _.isUndefined( raw['boundary.circle.radius'] ) ) {
-      // the default is small unless layers other than venue or address were explicitly specified
-      if (clean.layers && clean.layers.length > 0 && !_.includes(clean.layers, 'venue') && !_.includes(clean.layers, 'address')) {
-        raw['boundary.circle.radius'] = defaults['boundary:circle:radius:coarse'];
-      } else {
       raw['boundary.circle.radius'] = defaults['boundary:circle:radius'];
     }
-    }
 
     // santize the boundary.circle
     geo_common.sanitize_circle( 'boundary.circle', clean, raw, CIRCLE_IS_REQUIRED );

sanitizer/_geonames_deprecation.js

@@ -0,0 +1,25 @@
+const _ = require('lodash');
+
+/**
+with the release of coarse reverse as a separate thing and ES reverse only
+handling venues, addresses, and streets, geonames make no sense in the reverse context
+**/
+
+function sanitize( raw, clean, opts ) {
+  // error & warning messages
+  const messages = { errors: [], warnings: [] };
+
+  if (_.isEqual(clean.sources, ['geonames']) || _.isEqual(clean.sources, ['gn'])) {
+    messages.errors.push('/reverse does not support geonames');
+
+  } else if (_.includes(clean.sources, 'geonames') || _.includes(clean.sources, 'gn')) {
+    clean.sources = _.without(clean.sources, 'geonames', 'gn');
+    messages.warnings.push('/reverse does not support geonames');
+
+  }
+
+  return messages;
+  
+}
+
+module.exports = sanitize;

sanitizer/_geonames_warnings.js

@@ -0,0 +1,37 @@
+const _ = require('lodash');
+
+const non_admin_fields = ['number', 'street', 'query', 'category'];
+
+function hasAnyNonAdminFields(parsed_text) {
+  return !_.isEmpty(
+            _.intersection(
+              _.keys(parsed_text),
+              non_admin_fields));
+}
+
+function sanitize( raw, clean ){
+  // error & warning messages
+  const messages = { errors: [], warnings: [] };
+
+  // bail early if analysis isn't admin-only
+  if (_.isUndefined(clean.parsed_text) || hasAnyNonAdminFields(clean.parsed_text)) {
+    return messages;
+  }
+
+  // the analysis is admin-only, so add errors or warnings if geonames was requested
+  if (_.isEqual(clean.sources, ['geonames'])) {
+    // if requested sources is only geonames, return an error
+    messages.errors.push('input contains only administrative area data, ' +
+      'no results will be returned when sources=geonames');
+
+  } else if (_.includes(clean.sources, 'geonames')) {
+    // if there are other sources besides geonames, return an warning
+    messages.warnings.push('input contains only administrative area data, ' +
+      'geonames results will not be returned');
+
+  }
+
+  return messages;
+}
+
+module.exports = sanitize;

sanitizer/reverse.js

@@ -8,6 +8,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'),
       sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitizers, must be run after them
       sources_and_layers: require('../sanitizer/_sources_and_layers'),
+      geonames_deprecation: require('../sanitizer/_geonames_deprecation'),
       size: require('../sanitizer/_size')(/* use defaults*/),
       private: require('../sanitizer/_flag_bool')('private', false),
       geo_reverse: require('../sanitizer/_geo_reverse'),

sanitizer/search.js

@@ -15,7 +15,9 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'),
       private: require('../sanitizer/_flag_bool')('private', false),
       geo_search: require('../sanitizer/_geo_search'),
       boundary_country: require('../sanitizer/_boundary_country'),
-      categories: require('../sanitizer/_categories')
+      categories: require('../sanitizer/_categories'),
+      // this can go away once geonames has been abrogated
+      geonames_warnings: require('../sanitizer/_geonames_warnings')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };

schema.js

@@ -26,7 +26,20 @@ module.exports = Joi.object().keys({
     localization: Joi.object().keys({
       flipNumberAndStreetCountries: Joi.array().items(Joi.string().regex(/^[A-Z]{3}$/))
     }).unknown(false),
-    pipService: Joi.string().uri({ scheme: /https?/ })
+    pipService: Joi.any(), // got moved to services, ignored for now
+    placeholderService: Joi.any().forbidden(), // got moved to services
+    services: Joi.object().keys({
+      pip: Joi.object().keys({
+        url: Joi.string().uri({ scheme: /https?/ }),
+        timeout: Joi.number().integer().optional().default(250).min(0),
+        retries: Joi.number().integer().optional().default(3).min(0),
+      }).unknown(false).requiredKeys('url'),
+      placeholder: Joi.object().keys({
+        url: Joi.string().uri({ scheme: /https?/ }),
+        timeout: Joi.number().integer().optional().default(250).min(0),
+        retries: Joi.number().integer().optional().default(3).min(0),
+      }).unknown(false).requiredKeys('url')
+    }).unknown(false).default({}) // default api.services to an empty object
 
   }).requiredKeys('version', 'indexName', 'host').unknown(true),
   esclient: Joi.object().keys({

service/configurations/PlaceHolder.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class PlaceHolder extends ServiceConfiguration {
+  constructor(o) {
+    super('placeholder', o);
+  }
+
+  getParameters(req) {
+    const parameters = {
+      text: req.clean.text
+    };
+
+    if (_.has(req.clean, 'lang.iso6393')) {
+      parameters.lang = req.clean.lang.iso6393;
+    }
+
+    return parameters;
+  }
+
+  getUrl(req) {
+    return url.resolve(this.baseUrl, 'parser/search');
+  }
+
+}
+
+module.exports = PlaceHolder;

service/configurations/PointInPolygon.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const url = require('url');
+
+const _ = require('lodash');
+
+const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration;
+
+class PointInPolygon extends ServiceConfiguration {
+  constructor(o) {
+    super('pip', o);
+  }
+
+  getUrl(req) {
+    // use resolve to eliminate possibility of duplicate /'s in URL
+    return url.resolve(this.baseUrl, `${req.clean['point.lon']}/${req.clean['point.lat']}`);
+  }
+
+}
+
+module.exports = PointInPolygon;

service/pointinpolygon.js

@@ -1,86 +0,0 @@
-const logger = require( 'pelias-logger' ).get( 'pointinpolygon' );
-const request = require('request');
-const _ = require('lodash');
-
-function sanitizeUrl(requestUrl) {
-  return requestUrl.replace(/^(.+)\/.+\/.+$/, (match, base) => {
-    return `${base}/[removed]/[removed]`;
-  });
-}
-
-function parseErrorMessage(requestUrl, body, do_not_track) {
-  if (do_not_track) {
-    return `${sanitizeUrl(requestUrl)} returned status 200 but with non-JSON response: ${body}`;
-  }
-
-  return `${requestUrl} returned status 200 but with non-JSON response: ${body}`;
-
-}
-
-function serviceErrorMessage(requestUrl, statusCode, body, do_not_track) {
-  if (do_not_track) {
-    return `${sanitizeUrl(requestUrl)} returned status ${statusCode}: ${body}`;
-
-  } else {
-    return `${requestUrl} returned status ${statusCode}: ${body}`;
-
-  }
-
-}
-
-module.exports = (url) => {
-  if (!_.isEmpty(url)) {
-    logger.info(`using point-in-polygon service at ${url}`);
-
-    return function pointinpolygon( centroid, do_not_track, callback ) {
-      const requestUrl = `${url}/${centroid.lon}/${centroid.lat}`;
-
-      const options = {
-        method: 'GET',
-        url: requestUrl
-      };
-
-      if (do_not_track) {
-        options.headers = {
-          dnt: '1'
-        };
-      }
-
-      request(options, (err, response, body) => {
-        if (err) {
-          logger.error(JSON.stringify(err));
-          callback(err);
-        }
-        else if (response.statusCode === 200) {
-          try {
-            const parsed = JSON.parse(body);
-            callback(err, parsed);
-          }
-          catch (err) {
-            const parseMsg = parseErrorMessage(requestUrl, body, do_not_track);
-
-            logger.error(parseMsg);
-            callback(parseMsg);
-          }
-        }
-        else {
-          const errorMsg = serviceErrorMessage(requestUrl, response.statusCode, body, do_not_track);
-
-          logger.error(errorMsg);
-          callback(errorMsg);
-        }
-      });
-
-    };
-
-  } else {
-    logger.warn('point-in-polygon service disabled');
-
-    return (centroid, do_not_track, callback) => {
-      callback(`point-in-polygon service disabled, unable to resolve ` +
-        (do_not_track ? 'centroid' : JSON.stringify(centroid)));
-    };
-
-  }
-
-};

test/ciao/reverse/boundary_circle_valid_radius_coarse.coffee

@@ -0,0 +1,39 @@
+
+#> bounding circle
+path: '/v1/reverse?layers=coarse&point.lat=40.744243&point.lon=-73.990342&boundary.circle.radius=999.9'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.exist json.geocoding.warnings
+json.geocoding.warnings.should.eql [ 'boundary.circle.radius is not applicable for coarse reverse' ]
+
+#? inputs
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query['layers'].should.eql 'coarse'
+json.geocoding.query['point.lat'].should.eql 40.744243
+json.geocoding.query['point.lon'].should.eql -73.990342
+json.geocoding.query['boundary.circle.lat'].should.eql 40.744243
+json.geocoding.query['boundary.circle.lon'].should.eql -73.990342
+json.geocoding.query['boundary.circle.radius'].should.eql 999.9

test/unit/controller/coarse_reverse.js

@@ -2,6 +2,7 @@
 
 const setup = require('../../../controller/coarse_reverse');
 const proxyquire =  require('proxyquire').noCallThru();
+const _  = require('lodash');
 
 module.exports.tests = {};
 
@@ -15,12 +16,13 @@ module.exports.tests.interface = (test, common) => {
 
 module.exports.tests.early_exit_conditions = (test, common) => {
   test('should_execute returning false should not call service', (t) => {
+    t.plan(2);
+
     const service = () => {
       throw Error('service should not have been called');
     };
 
-    const should_execute = () => { return false; };
+    const controller = setup(service, _.constant(false));
-    const controller = setup(service, should_execute);
 
     const req = {
       clean: {
@@ -30,14 +32,12 @@ module.exports.tests.early_exit_conditions = (test, common) => {
     };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     // passing res=undefined verifies that it wasn't interacted with
     t.doesNotThrow(controller.bind(null, req, undefined, next));
-    t.ok(next_was_called);
     t.end();
 
   });
@@ -46,44 +46,108 @@ module.exports.tests.early_exit_conditions = (test, common) => {
 
 module.exports.tests.error_conditions = (test, common) => {
   test('service error should log and call next', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(3);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['locality'] } } );
       callback('this is an error');
     };
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
-          return 'do_not_track value';
-        }
-      }
-    })(service, should_execute);
 
     const req = {
       clean: {
-        layers: ['locality'],
+        layers: ['locality']
-        point: {
-          lat: 12.121212,
-          lon: 21.212121
-        }
       }
     };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     // passing res=undefined verifies that it wasn't interacted with
     controller(req, undefined, next);
 
     t.ok(logger.isErrorMessage('this is an error'));
-    t.ok(next_was_called);
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.boundary_circle_radius_warnings = (test, common) => {
+  test('defined clean[boundary.circle.radius] should add a warning', (t) => {
+    const service = (req, callback) => {
+      callback(undefined, {});
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../controller/coarse_reverse', {
+      'pelias-logger': logger
+    })(service, _.constant(true));
+
+    const req = {
+      warnings: [],
+      clean: {
+        'boundary.circle.radius': 17
+      }
+    };
+
+    const res = { };
+
+    const next = () => {};
+
+    controller(req, res, next);
+
+    const expected = {
+      meta: {},
+      data: []
+    };
+
+    t.deepEquals(req.warnings, ['boundary.circle.radius is not applicable for coarse reverse']);
+    t.deepEquals(res, expected);
+    t.notOk(logger.hasErrorMessages());
+    t.end();
+
+  });
+
+  test('defined clean[boundary.circle.radius] should add a warning', (t) => {
+    const service = (req, callback) => {
+      callback(undefined, {});
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../controller/coarse_reverse', {
+      'pelias-logger': logger
+    })(service, _.constant(true));
+
+    const req = {
+      warnings: [],
+      clean: {}
+    };
+
+    const res = { };
+
+    // verify that next was called
+    const next = () => { };
+
+    controller(req, res, next);
+
+    const expected = {
+      meta: {},
+      data: []
+    };
+
+    t.deepEquals(req.warnings, []);
+    t.deepEquals(res, expected);
+    t.notOk(logger.hasErrorMessages());
     t.end();
 
   });
@@ -92,8 +156,11 @@ module.exports.tests.error_conditions = (test, common) => {
 
 module.exports.tests.success_conditions = (test, common) => {
   test('service returning results should use first entry for each layer', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(4);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['neighbourhood'] } } );
+
       const results = {
         neighbourhood: [
           {
@@ -151,32 +218,21 @@ module.exports.tests.success_conditions = (test, common) => {
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
-          return 'do_not_track value';
-        }
-      }
-    })(service, should_execute);
 
     const req = {
       clean: {
-        layers: ['neighbourhood'],
+        layers: ['neighbourhood']
-        point: {
-          lat: 12.121212,
-          lon: 21.212121
-        }
       }
     };
 
     const res = { };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     controller(req, res, next);
@@ -228,7 +284,6 @@ module.exports.tests.success_conditions = (test, common) => {
             country_id: ['100'],
             country_a: ['xyz']
           },
-          alpha3: 'XYZ',
           center_point: {
             lat: 12.121212,
             lon: 21.212121
@@ -239,22 +294,22 @@ module.exports.tests.success_conditions = (test, common) => {
     };
 
     t.deepEquals(res, expected);
-
     t.notOk(logger.hasErrorMessages());
-    t.ok(next_was_called);
     t.end();
 
   });
 
   test('layers missing from results should be ignored', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(4);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['neighbourhood'] } } );
+
       const results = {
         neighbourhood: [
           {
             id: 10,
             name: 'neighbourhood name',
-            abbr: 'neighbourhood abbr',
             centroid: {
               lat: 12.121212,
               lon: 21.212121
@@ -269,32 +324,21 @@ module.exports.tests.success_conditions = (test, common) => {
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
-          return 'do_not_track value';
-        }
-      }
-    })(service, should_execute);
 
     const req = {
       clean: {
-        layers: ['neighbourhood'],
+        layers: ['neighbourhood']
-        point: {
-          lat: 12.121212,
-          lon: 21.212121
-        }
       }
     };
 
     const res = { };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     controller(req, res, next);
@@ -317,7 +361,7 @@ module.exports.tests.success_conditions = (test, common) => {
           parent: {
             neighbourhood: ['neighbourhood name'],
             neighbourhood_id: ['10'],
-            neighbourhood_a: ['neighbourhood abbr']
+            neighbourhood_a: [null]
           },
           center_point: {
             lat: 12.121212,
@@ -329,16 +373,17 @@ module.exports.tests.success_conditions = (test, common) => {
     };
 
     t.deepEquals(res, expected);
-
     t.notOk(logger.hasErrorMessages());
-    t.ok(next_was_called);
     t.end();
 
   });
 
   test('most granular layer missing centroid should not set', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(4);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['neighbourhood'] } } );
+
       const results = {
         neighbourhood: [
           {
@@ -355,32 +400,21 @@ module.exports.tests.success_conditions = (test, common) => {
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
-          return 'do_not_track value';
-        }
-      }
-    })(service, should_execute);
 
     const req = {
       clean: {
-        layers: ['neighbourhood'],
+        layers: ['neighbourhood']
-        point: {
-          lat: 12.121212,
-          lon: 21.212121
-        }
       }
     };
 
     const res = { };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     controller(req, res, next);
@@ -411,16 +445,17 @@ module.exports.tests.success_conditions = (test, common) => {
     };
 
     t.deepEquals(res, expected);
-
     t.notOk(logger.hasErrorMessages());
-    t.ok(next_was_called);
     t.end();
 
   });
 
   test('most granular layer missing bounding_box should not set', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(4);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['neighbourhood'] } } );
+
       const results = {
         neighbourhood: [
           {
@@ -440,19 +475,91 @@ module.exports.tests.success_conditions = (test, common) => {
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
+
-          return 'do_not_track value';
+    const req = {
+      clean: {
+        layers: ['neighbourhood']
+      }
+    };
+
+    const res = { };
+
+    // verify that next was called
+    const next = () => {
+      t.pass('next() was called');
+    };
+
+    controller(req, res, next);
+
+    const expected = {
+      meta: {},
+      data: [
+        {
+          _id: '10',
+          _type: 'neighbourhood',
+          layer: 'neighbourhood',
+          source: 'whosonfirst',
+          source_id: '10',
+          name: {
+            'default': 'neighbourhood name'
+          },
+          phrase: {
+            'default': 'neighbourhood name'
+          },
+          parent: {
+            neighbourhood: ['neighbourhood name'],
+            neighbourhood_id: ['10'],
+            neighbourhood_a: ['neighbourhood abbr']
+          },
+          center_point: {
+            lat: 12.121212,
+            lon: 21.212121
+          }
         }
+      ]
+    };
+
+    t.deepEquals(res, expected);
+
+    t.notOk(logger.hasErrorMessages());
+    t.end();
+
+  });
+
+  test('no requested layers should use everything', (t) => {
+    // this test is used to test coarse reverse fallback for when non-coarse reverse
+    //  was requested but no non-coarse results were found
+
+    // by plan'ing the number of tests, we can verify that next() was called w/o
+    //  additional bookkeeping
+    t.plan(4);
+
+    const service = (req, callback) => {
+      const results = {
+        neighbourhood: [
+          {
+            id: 10,
+            name: 'neighbourhood name',
+            abbr: 'neighbourhood abbr'
           }
-    })(service, should_execute);
+        ]
+      };
+
+      callback(undefined, results);
+    };
+
+    const logger = require('pelias-mock-logger')();
+
+    const controller = proxyquire('../../../controller/coarse_reverse', {
+      'pelias-logger': logger
+    })(service, _.constant(true));
 
     const req = {
       clean: {
-        layers: ['neighbourhood'],
+        layers: [],
         point: {
           lat: 12.121212,
           lon: 21.212121
@@ -463,9 +570,8 @@ module.exports.tests.success_conditions = (test, common) => {
     const res = { };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() should have been called');
     };
 
     controller(req, res, next);
@@ -489,19 +595,183 @@ module.exports.tests.success_conditions = (test, common) => {
             neighbourhood: ['neighbourhood name'],
             neighbourhood_id: ['10'],
             neighbourhood_a: ['neighbourhood abbr']
-          },
+          }
-          center_point: {
+        }
+      ]
+    };
+
+    t.deepEquals(req.clean.layers, [], 'req.clean.layers should be unmodified');
+    t.deepEquals(res, expected);
+    t.notOk(logger.hasErrorMessages());
+
+    t.end();
+
+  });
+
+  test('layers specifying only venue, address, or street should not exclude coarse results', (t) => {
+    // this test is used to test coarse reverse fallback for when non-coarse reverse
+    //  was requested but no non-coarse results were found
+    const non_coarse_layers = ['venue', 'address', 'street'];
+    const tests_per_non_coarse_layer = 4;
+
+    // by plan'ing the number of tests, we can verify that next() was called w/o
+    //  additional bookkeeping
+    t.plan(non_coarse_layers.length * tests_per_non_coarse_layer);
+
+    non_coarse_layers.forEach((non_coarse_layer) => {
+      const service = (req, callback) => {
+        const results = {
+          neighbourhood: [
+            {
+              id: 10,
+              name: 'neighbourhood name',
+              abbr: 'neighbourhood abbr'
+            }
+          ]
+        };
+
+        callback(undefined, results);
+      };
+
+      const logger = require('pelias-mock-logger')();
+
+      const controller = proxyquire('../../../controller/coarse_reverse', {
+        'pelias-logger': logger
+      })(service, _.constant(true));
+
+      const req = {
+        clean: {
+          layers: [non_coarse_layer],
+          point: {
             lat: 12.121212,
             lon: 21.212121
           }
         }
+      };
+
+      const res = { };
+
+      // verify that next was called
+      const next = () => {
+        t.pass('next() should have been called');
+      };
+
+      controller(req, res, next);
+
+      const expected = {
+        meta: {},
+        data: [
+          {
+            _id: '10',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '10',
+            name: {
+              'default': 'neighbourhood name'
+            },
+            phrase: {
+              'default': 'neighbourhood name'
+            },
+            parent: {
+              neighbourhood: ['neighbourhood name'],
+              neighbourhood_id: ['10'],
+              neighbourhood_a: ['neighbourhood abbr']
+            }
+          }
         ]
       };
 
+      t.deepEquals(req.clean.layers, [non_coarse_layer], 'req.clean.layers should be unmodified');
       t.deepEquals(res, expected);
+      t.notOk(logger.hasErrorMessages());
+
+    });
 
+    t.end();
+
+  });
+
+  test('layers specifying venue, address, or street AND coarse layer should not exclude coarse results', (t) => {
+    // this test is used to test coarse reverse fallback for when non-coarse reverse
+    //  was requested but no non-coarse results were found
+    const non_coarse_layers = ['venue', 'address', 'street'];
+    const tests_per_non_coarse_layer = 4;
+
+    // by plan'ing the number of tests, we can verify that next() was called w/o
+    //  additional bookkeeping
+    t.plan(non_coarse_layers.length * tests_per_non_coarse_layer);
+
+    non_coarse_layers.forEach((non_coarse_layer) => {
+      const service = (req, callback) => {
+        const results = {
+          neighbourhood: [
+            {
+              id: 10,
+              name: 'neighbourhood name',
+              abbr: 'neighbourhood abbr'
+            }
+          ]
+        };
+
+        callback(undefined, results);
+      };
+
+      const logger = require('pelias-mock-logger')();
+
+      const controller = proxyquire('../../../controller/coarse_reverse', {
+        'pelias-logger': logger
+      })(service, _.constant(true));
+
+      const req = {
+        clean: {
+          layers: [non_coarse_layer, 'neighbourhood'],
+          point: {
+            lat: 12.121212,
+            lon: 21.212121
+          }
+        }
+      };
+
+      const res = { };
+
+      // verify that next was called
+      const next = () => {
+        t.pass('next() should have been called');
+      };
+
+      controller(req, res, next);
+
+      const expected = {
+        meta: {},
+        data: [
+          {
+            _id: '10',
+            _type: 'neighbourhood',
+            layer: 'neighbourhood',
+            source: 'whosonfirst',
+            source_id: '10',
+            name: {
+              'default': 'neighbourhood name'
+            },
+            phrase: {
+              'default': 'neighbourhood name'
+            },
+            parent: {
+              neighbourhood: ['neighbourhood name'],
+              neighbourhood_id: ['10'],
+              neighbourhood_a: ['neighbourhood abbr']
+            }
+          }
+        ]
+      };
+
+      t.deepEquals(req.clean.layers, [non_coarse_layer, 'neighbourhood'], 'req.clean.layers should be unmodified');
+      t.deepEquals(res, expected);
       t.notOk(logger.hasErrorMessages());
-    t.ok(next_was_called);
+
+    });
+
     t.end();
 
   });
@@ -510,8 +780,11 @@ module.exports.tests.success_conditions = (test, common) => {
 
 module.exports.tests.failure_conditions = (test, common) => {
   test('service returning 0 results at the requested layer should return nothing', (t) => {
-    const service = (point, do_not_track, callback) => {
+    t.plan(4);
-      t.equals(do_not_track, 'do_not_track value');
+
+    const service = (req, callback) => {
+      t.deepEquals(req, { clean: { layers: ['neighbourhood'] } } );
+
       // response without neighbourhood results
       const results = {
         borough: [
@@ -557,32 +830,21 @@ module.exports.tests.failure_conditions = (test, common) => {
 
     const logger = require('pelias-mock-logger')();
 
-    const should_execute = () => { return true; };
     const controller = proxyquire('../../../controller/coarse_reverse', {
-      'pelias-logger': logger,
+      'pelias-logger': logger
-      '../helper/logging': {
+    })(service, _.constant(true));
-        isDNT: () => {
-          return 'do_not_track value';
-        }
-      }
-    })(service, should_execute);
 
     const req = {
       clean: {
-        layers: ['neighbourhood'],
+        layers: ['neighbourhood']
-        point: {
-          lat: 12.121212,
-          lon: 21.212121
-        }
       }
     };
 
     const res = { };
 
     // verify that next was called
-    let next_was_called = false;
     const next = () => {
-      next_was_called = true;
+      t.pass('next() was called');
     };
 
     controller(req, res, next);
@@ -593,9 +855,7 @@ module.exports.tests.failure_conditions = (test, common) => {
     };
 
     t.deepEquals(res, expected);
-
     t.notOk(logger.hasErrorMessages());
-    t.ok(next_was_called);
     t.end();
 
   });

test/unit/controller/predicates/has_results_at_layers.js

@@ -0,0 +1,122 @@
+'use strict';
+
+const _ = require('lodash');
+const has_results_at_layers = require('../../../../controller/predicates/has_results_at_layers');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.equal(typeof has_results_at_layers, 'function', 'has_results_at_layers is a function');
+    t.equal(has_results_at_layers.length, 1);
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('should return true when any result.layer matches any layer in array', (t) => {
+    const req = {};
+    const res = {
+      data: [
+        {
+          layer: 'layer 1'
+        },
+        {
+          layer: 'layer 2'
+        },
+        {
+          layer: 'layer 3'
+        }
+      ]
+    };
+
+    t.ok(has_results_at_layers(['layer 2', 'layer 4'])(req, res));
+    t.end();
+
+  });
+
+  test('should return true when any result.layer matches layer string', (t) => {
+    const req = {};
+    const res = {
+      data: [
+        {
+          layer: 'layer 1'
+        },
+        {
+          layer: 'layer 2'
+        },
+        {
+          layer: 'layer 3'
+        }
+      ]
+    };
+
+    t.ok(has_results_at_layers('layer 2')(req, res));
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('should return false when response has undefined data', (t) => {
+    const req = {};
+    const res = {};
+
+    t.notOk(has_results_at_layers('layer')(req, res));
+    t.end();
+
+  });
+
+  test('should return false when response has empty data array', (t) => {
+    const req = {};
+    const res = {
+      data: []
+    };
+
+    t.notOk(has_results_at_layers('layer')(req, res));
+    t.end();
+
+  });
+
+  test('should return false when layer is a substring of non-array string layers parameter', (t) => {
+    const req = {};
+    const res = {
+      data: [
+        {
+          layer: 'aye'
+        }
+      ]
+    };
+
+    t.notOk(has_results_at_layers('layer')(req, res));
+    t.end();
+
+  });
+
+  test('should return false when no results have layer in supplied layers', (t) => {
+    const req = {};
+    const res = {
+      data: [
+        {
+          layer: 'layer 1'
+        }
+      ]
+    };
+
+    t.notOk(has_results_at_layers(['layer 2', 'layer 3'])(req, res));
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /has_results_at_layers ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_admin_only_analysis.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const _ = require('lodash');
+const is_admin_only_analysis = require('../../../../controller/predicates/is_admin_only_analysis');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = (test, common) => {
+  test('valid interface', (t) => {
+    t.equal(typeof is_admin_only_analysis, 'function', 'is_admin_only_analysis is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.true_conditions = (test, common) => {
+  test('parsed_text with admin-only properties should return true', (t) => {
+    ['neighbourhood', 'borough', 'city', 'county', 'state', 'postalcode', 'country'].forEach((property) => {
+      const req = {
+        clean: {
+          parsed_text: {}
+        }
+      };
+      const res = {};
+
+      req.clean.parsed_text[property] = `${property} value`;
+
+      t.ok(is_admin_only_analysis(req, res));
+
+    });
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.false_conditions = (test, common) => {
+  test('req.clean with no parsed_text should return false', (t) => {
+    const req = {
+      clean: {
+      }
+    };
+    const res = {};
+
+    t.notOk(is_admin_only_analysis(req, res));
+    t.end();
+
+  });
+
+  test('parsed_text with non-admin properties should return false', (t) => {
+    ['number', 'street', 'query', 'category'].forEach((property) => {
+      const req = {
+        clean: {
+          parsed_text: {}
+        }
+      };
+      const res = {};
+
+      req.clean.parsed_text[property] = `${property} value`;
+
+      t.notOk(is_admin_only_analysis(req, res));
+
+    });
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`GET /is_admin_only_analysis ${name}`, testFunction);
+  }
+
+  for( const testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/controller/predicates/is_service_enabled.js

@@ -1,21 +1,21 @@
 'use strict';
 
 const _ = require('lodash');
-const is_pip_service_enabled = require('../../../../controller/predicates/is_pip_service_enabled');
+const is_service_enabled = require('../../../../controller/predicates/is_service_enabled');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = (test, common) => {
   test('valid interface', (t) => {
-    t.equal(typeof is_pip_service_enabled, 'function', 'is_pip_service_enabled  is a function');
+    t.equal(typeof is_service_enabled, 'function', 'is_service_enabled  is a function');
-    t.equal(typeof is_pip_service_enabled(), 'function', 'is_pip_service_enabled() is a function');
+    t.equal(typeof is_service_enabled(), 'function', 'is_service_enabled() is a function');
     t.end();
   });
 };
 
 module.exports.tests.true_conditions = (test, common) => {
   test('string uri should return true', (t) => {
-    t.ok(is_pip_service_enabled('pip uri')());
+    t.ok(is_service_enabled('pip uri')());
     t.end();
 
   });
@@ -24,7 +24,7 @@ module.exports.tests.true_conditions = (test, common) => {
 
 module.exports.tests.false_conditions = (test, common) => {
   test('undefined uri should return false', (t) => {
-    t.notOk(is_pip_service_enabled()());
+    t.notOk(is_service_enabled()());
     t.end();
 
   });
@@ -33,7 +33,7 @@ module.exports.tests.false_conditions = (test, common) => {
 
 module.exports.all = (tape, common) => {
   function test(name, testFunction) {
-    return tape(`GET /is_pip_service_enabled ${name}`, testFunction);
+    return tape(`GET /is_service_enabled ${name}`, testFunction);
   }
 
   for( const testCase in module.exports.tests ){

test/unit/fixture/reverse_null_island.js

@@ -5,7 +5,7 @@ module.exports = {
     'bool': {
       'filter': [{
         'geo_distance': {
-          'distance': '500km',
+          'distance': '3km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
           'center_point': {
@@ -13,6 +13,11 @@ module.exports = {
             'lon': 0
           }
         }
+      },
+      {
+        'terms': {
+          'layer': ['venue', 'address', 'street']
+        }
       }]
     }
   },

test/unit/fixture/reverse_standard.js

@@ -5,7 +5,7 @@ module.exports = {
     'bool': {
       'filter': [{
         'geo_distance': {
-          'distance': '500km',
+          'distance': '3km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
           'center_point': {
@@ -13,6 +13,11 @@ module.exports = {
             'lon': -82.50622
           }
         }
+      },
+      {
+        'terms': {
+          'layer': ['venue', 'address', 'street']
+        }
       }]
     }
   },

test/unit/fixture/reverse_with_boundary_country.js

@@ -15,7 +15,7 @@ module.exports = {
       ],
       'filter': [{
         'geo_distance': {
-          'distance': '500km',
+          'distance': '3km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
           'center_point': {
@@ -23,6 +23,11 @@ module.exports = {
             'lon': -82.50622
           }
         }
+      },
+      {
+        'terms': {
+          'layer': ['venue', 'address', 'street']
+        }
       }]
     }
   },

test/unit/fixture/reverse_with_layer_filtering.js

@@ -6,7 +6,7 @@ module.exports = {
       'filter': [
         {
           'geo_distance': {
-            'distance': '500km',
+            'distance': '3km',
             'distance_type': 'plane',
             'optimize_bbox': 'indexed',
             'center_point': {
@@ -17,7 +17,7 @@ module.exports = {
         },
         {
           'terms': {
-            'layer': ['country']
+            'layer': ['venue', 'address', 'street']
           }
         }
       ]

test/unit/fixture/reverse_with_layer_filtering_non_coarse_subset.js

@@ -0,0 +1,41 @@
+var vs = require('../../../query/reverse_defaults');
+
+module.exports = {
+  'query': {
+    'bool': {
+      'filter': [
+        {
+          'geo_distance': {
+            'distance': '3km',
+            'distance_type': 'plane',
+            'optimize_bbox': 'indexed',
+            'center_point': {
+              'lat': 29.49136,
+              'lon': -82.50622
+            }
+          }
+        },
+        {
+          'terms': {
+            'layer': ['venue', 'street']
+          }
+        }
+      ]
+    }
+  },
+  'sort': [
+    '_score',
+    {
+      '_geo_distance': {
+        'center_point': {
+          'lat': 29.49136,
+          'lon': -82.50622
+        },
+        'order': 'asc',
+        'distance_type': 'plane'
+      }
+    }
+  ],
+  'size': vs.size,
+  'track_scores': true
+};

test/unit/fixture/reverse_with_source_filtering.js

@@ -6,7 +6,7 @@ module.exports = {
       'filter': [
         {
           'geo_distance': {
-            'distance': '500km',
+            'distance': '3km',
             'distance_type': 'plane',
             'optimize_bbox': 'indexed',
             'center_point': {
@@ -19,6 +19,11 @@ module.exports = {
           'terms': {
             'source': ['test']
           }
+        },
+        {
+          'terms': {
+            'layer': ['venue', 'address', 'street']
+          }
         }
       ]
     }

test/unit/middleware/sortResponseData.js

@@ -0,0 +1,156 @@
+const _ = require('lodash');
+const proxyquire =  require('proxyquire').noCallThru();
+const mock_logger = require('pelias-mock-logger');
+
+const sortResponseData = require('../../../middleware/sortResponseData');
+
+module.exports.tests = {};
+
+module.exports.tests.should_execute_failure = (test, common) => {
+  test('should_execute returning false should call next w/o invoking comparator', (t) => {
+    t.plan(2, 'this ensures that should_execute was invoked');
+
+    const comparator = () => {
+      throw Error('should not have been called');
+    };
+
+    const should_execute = (req, res) => {
+      t.deepEquals(req, { a: 1 });
+      t.deepEquals(res, { b: 2 });
+      return false;
+    };
+
+    const sort = sortResponseData(comparator, should_execute);
+
+    const req = { a: 1 };
+    const res = { b: 2 };
+
+    sort(req, res, () => {
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.tests.general_tests = (test, common) => {
+  test('req.clean should be passed to sort', (t) => {
+    t.plan(1, 'this ensures that comparator was invoked');
+
+    const comparator = (clean) => {
+      t.deepEquals(clean, { a: 1 });
+      return () => {
+        throw Error('should not have been called');
+      };
+    };
+
+    const sort = sortResponseData(comparator, _.constant(true));
+
+    const req = {
+      clean: {
+        a: 1
+      }
+    };
+
+    const res = {
+      data: [ {} ]
+    };
+
+    sort(req, res, () => {
+      t.end();
+    });
+
+  });
+
+  test('undefined res.data should return without interacting with comparator', (t) => {
+    const comparator = () => {
+      throw Error('should not have been called');
+    };
+
+    const sort = sortResponseData(comparator, _.constant(true));
+
+    const req = {};
+    const res = {};
+
+    sort(req, res, () => {
+      t.deepEquals(res, {});
+      t.end();
+    });
+
+  });
+
+  test('empty res.data should return without interacting with comparator', (t) => {
+    const comparator = () => {
+      throw Error('should not have been called');
+    };
+
+    const sort = sortResponseData(comparator, _.constant(true));
+
+    const req = {};
+    const res = {
+      data: []
+    };
+
+    sort(req, res, () => {
+      t.deepEquals(res.data, [], 'res.data should still be empty');
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.tests.successful_sort = (test, common) => {
+  test('comparator should be sort res.data', (t) => {
+    const logger = mock_logger();
+
+    const comparator = () => {
+      return (a, b) => {
+        return a._id > b._id;
+      };
+    };
+
+    const sortResponseData = proxyquire('../../../middleware/sortResponseData', {
+      'pelias-logger': logger
+    });
+
+    const sort = sortResponseData(comparator, _.constant(true));
+
+    const req = {
+      clean: {
+        field: 'value'
+      }
+    };
+    const res = {
+      data: [
+        { _id: 3 },
+        { _id: 2 },
+        { _id: 1 },
+      ]
+    };
+
+    sort(req, res, () => {
+      t.deepEquals(res.data.shift(), { _id: 1 });
+      t.deepEquals(res.data.shift(), { _id: 2 });
+      t.deepEquals(res.data.shift(), { _id: 3 });
+
+      t.ok(logger.isDebugMessage(
+        'req.clean: {"field":"value"}, pre-sort: [3,2,1], post-sort: [1,2,3]'));
+
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+
+  function test(name, testFunction) {
+    return tape(`[middleware] sortResponseData: ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/reverse.js

@@ -16,7 +16,7 @@ module.exports.tests.query = function(test, common) {
       'point.lon': -82.50622,
       'boundary.circle.lat': 29.49136,
       'boundary.circle.lon': -82.50622,
-      'boundary.circle.radius': 500
+      'boundary.circle.radius': 3
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
@@ -33,7 +33,7 @@ module.exports.tests.query = function(test, common) {
       'point.lon': 0,
       'boundary.circle.lat': 0,
       'boundary.circle.lon': 0,
-      'boundary.circle.radius': 500
+      'boundary.circle.radius': 3
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
@@ -67,7 +67,7 @@ module.exports.tests.query = function(test, common) {
       'point.lon': -82.50622,
       'boundary.circle.lat': 111,
       'boundary.circle.lon': 333,
-      'boundary.circle.radius': 500
+      'boundary.circle.radius': 3
     };
     var query = generate(clean);
     var compiled = JSON.parse( JSON.stringify( query ) );
@@ -102,7 +102,7 @@ module.exports.tests.query = function(test, common) {
       'point.lon': -82.50622,
       'boundary.circle.lat': 29.49136,
       'boundary.circle.lon': -82.50622,
-      'boundary.circle.radius': 500,
+      'boundary.circle.radius': 3,
       'boundary.country': 'ABC'
     });
 
@@ -120,7 +120,7 @@ module.exports.tests.query = function(test, common) {
       'point.lon': -82.50622,
       'boundary.circle.lat': 29.49136,
       'boundary.circle.lon': -82.50622,
-      'boundary.circle.radius': 500,
+      'boundary.circle.radius': 3,
       'sources': ['test']
     });
 
@@ -132,23 +132,46 @@ module.exports.tests.query = function(test, common) {
     t.end();
   });
 
-  test('valid layers filter', function(t) {
+  test('valid layers filter', (t) => {
-    var query = generate({
+    const query = generate({
       'point.lat': 29.49136,
       'point.lon': -82.50622,
       'boundary.circle.lat': 29.49136,
       'boundary.circle.lon': -82.50622,
-      'boundary.circle.radius': 500,
+      'boundary.circle.radius': 3,
-      'layers': ['country']
+      // only venue, address, and street layers should be retained
+      'layers': ['neighbourhood', 'venue', 'locality', 'address', 'region', 'street', 'country']
     });
 
-    var compiled = JSON.parse( JSON.stringify( query ) );
+    const compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/reverse_with_layer_filtering');
+    const expected = require('../fixture/reverse_with_layer_filtering');
 
     t.deepEqual(compiled.type, 'reverse', 'query type set');
     t.deepEqual(compiled.body, expected, 'valid reverse query with source filtering');
     t.end();
+
+  });
+
+  test('valid layers filter - subset of non-coarse layers', (t) => {
+    const query = generate({
+      'point.lat': 29.49136,
+      'point.lon': -82.50622,
+      'boundary.circle.lat': 29.49136,
+      'boundary.circle.lon': -82.50622,
+      'boundary.circle.radius': 3,
+      // only venue, address, and street layers should be retained
+      'layers': ['neighbourhood', 'venue', 'street', 'locality']
     });
+
+    const compiled = JSON.parse( JSON.stringify( query ) );
+    const expected = require('../fixture/reverse_with_layer_filtering_non_coarse_subset');
+
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid reverse query with source filtering');
+    t.end();
+
+  });
+
 };
 
 module.exports.all = function (tape, common) {

test/unit/run.js

@@ -14,11 +14,14 @@ var tests = [
   require('./controller/coarse_reverse'),
   require('./controller/index'),
   require('./controller/place'),
+  require('./controller/placeholder'),
   require('./controller/search'),
   require('./controller/predicates/has_response_data'),
+  require('./controller/predicates/has_results_at_layers'),
   require('./controller/predicates/has_request_errors'),
+  require('./controller/predicates/is_admin_only_analysis'),
   require('./controller/predicates/is_coarse_reverse'),
-  require('./controller/predicates/is_pip_service_enabled'),
+  require('./controller/predicates/is_service_enabled'),
   require('./helper/diffPlaces'),
   require('./helper/geojsonify'),
   require('./helper/logging'),
@@ -38,6 +41,7 @@ var tests = [
   require('./middleware/parseBBox'),
   require('./middleware/sendJSON'),
   require('./middleware/normalizeParentIds'),
+  require('./middleware/sortResponseData'),
   require('./middleware/trimByGranularity'),
   require('./middleware/trimByGranularityStructured'),
   require('./middleware/requestLanguage'),
@@ -52,6 +56,8 @@ var tests = [
   require('./query/text_parser'),
   require('./sanitizer/_boundary_country'),
   require('./sanitizer/_flag_bool'),
+  require('./sanitizer/_geonames_deprecation'),
+  require('./sanitizer/_geonames_warnings'),
   require('./sanitizer/_geo_common'),
   require('./sanitizer/_geo_reverse'),
   require('./sanitizer/_groups'),
@@ -78,10 +84,11 @@ var tests = [
   require('./sanitizer/search'),
   require('./sanitizer/search_fallback'),
   require('./sanitizer/wrap'),
+  require('./service/configurations/PlaceHolder'),
+  require('./service/configurations/PointInPolygon'),
   require('./service/mget'),
   require('./service/search'),
   require('./service/interpolation'),
-  require('./service/pointinpolygon'),
   require('./service/language')
 ];
 

test/unit/sanitizer/_geo_reverse.js

@@ -1,17 +1,19 @@
-var sanitize = require('../../../sanitizer/_geo_reverse');
+'use strict';
-var defaults = require('../../../query/reverse_defaults');
+
+const sanitize = require('../../../sanitizer/_geo_reverse');
+const defaults = require('../../../query/reverse_defaults');
 
 module.exports.tests = {};
 
-module.exports.tests.sanitize_boundary_country = function(test, common) {
+module.exports.tests.warning_situations = (test, common) => {
-  test('raw with boundary.circle.lat should add warning about ignored boundary.circle', function(t) {
+  test('raw with boundary.circle.lat should add warning about ignored boundary.circle', (t) => {
-    var raw = {
+    const raw = {
       'point.lat': '12.121212',
       'point.lon': '21.212121',
       'boundary.circle.lat': '13.131313'
     };
-    var clean = {};
+    const clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitize(raw, clean);
 
     t.equals(clean['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
     t.deepEquals(errorsAndWarnings, {
@@ -19,16 +21,17 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
       warnings: ['boundary.circle.lat/boundary.circle.lon are currently unsupported']
     }, 'no warnings/errors');
     t.end();
+
   });
 
-  test('raw with boundary.circle.lon should add warning about ignored boundary.circle', function(t) {
+  test('raw with boundary.circle.lon should add warning about ignored boundary.circle', (t) => {
-    var raw = {
+    const raw = {
       'point.lat': '12.121212',
       'point.lon': '21.212121',
       'boundary.circle.lon': '31.313131'
     };
-    var clean = {};
+    const clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitize(raw, clean);
 
     t.equals(clean['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
     t.deepEquals(errorsAndWarnings, {
@@ -36,16 +39,17 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
       warnings: ['boundary.circle.lat/boundary.circle.lon are currently unsupported']
     }, 'no warnings/errors');
     t.end();
+
   });
 
-  test('raw with boundary.circle.radius shouldn\'t add warning about ignored boundary.circle', function(t) {
+  test('raw with boundary.circle.radius shouldn\'t add warning about ignored boundary.circle', (t) => {
-    var raw = {
+    const raw = {
       'point.lat': '12.121212',
       'point.lon': '21.212121',
       'boundary.circle.radius': '17'
     };
-    var clean = {};
+    const clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitize(raw, clean);
 
     // t.equals(clean['boundary.circle.radius'], 12.121212, 'should be set to point.lat')
     t.deepEquals(errorsAndWarnings, {
@@ -53,85 +57,61 @@ module.exports.tests.sanitize_boundary_country = function(test, common) {
       warnings: []
     }, 'no warnings/errors');
     t.end();
+
   });
 
-  test('boundary.circle.lat/lon should be overridden with point.lat/lon', function(t) {
+};
-    var raw = {
+
+module.exports.tests.success_conditions = (test, common) => {
+  test('boundary.circle.radius specified in request should override default', (t) => {
+    const raw = {
       'point.lat': '12.121212',
       'point.lon': '21.212121',
-      'boundary.circle.lat': '13.131313',
+      'boundary.circle.radius': '3248732857km' // this will never be the default
-      'boundary.circle.lon': '31.313131'
     };
-    var clean = {};
+    const clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitize(raw, clean);
 
-    t.equals(raw['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
+    t.equals(raw['boundary.circle.lat'], 12.121212);
-    t.equals(raw['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
+    t.equals(raw['boundary.circle.lon'], 21.212121);
-    t.equals(clean['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
+    t.equals(raw['boundary.circle.radius'], '3248732857km');
-    t.equals(clean['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
+    t.equals(clean['boundary.circle.lat'], 12.121212);
+    t.equals(clean['boundary.circle.lon'], 21.212121);
+    t.equals(clean['boundary.circle.radius'], 3248732857.0);
+
+    t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] });
     t.end();
+
   });
 
-  test('no boundary.circle.radius and no layers supplied should be set to default', function(t) {
+  test('boundary.circle.radius not specified should use default', (t) => {
-    var raw = {
+    const raw = {
       'point.lat': '12.121212',
       'point.lon': '21.212121'
     };
-    var clean = {};
+    const clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
+    const errorsAndWarnings = sanitize(raw, clean);
 
+    t.equals(raw['boundary.circle.lat'], 12.121212);
+    t.equals(raw['boundary.circle.lon'], 21.212121);
     t.equals(raw['boundary.circle.radius'], defaults['boundary:circle:radius'], 'should be from defaults');
+    t.equals(clean['boundary.circle.lat'], 12.121212);
+    t.equals(clean['boundary.circle.lon'], 21.212121);
     t.equals(clean['boundary.circle.radius'], parseFloat(defaults['boundary:circle:radius']), 'should be same as raw');
-    t.end();
-  });
 
-  test('no boundary.circle.radius and coarse layers supplied should be set to coarse default', function(t) {
+    t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] });
-    var raw = {
-      'point.lat': '12.121212',
-      'point.lon': '21.212121'
-    };
-    var clean = { layers: 'coarse' };
-    var errorsAndWarnings = sanitize(raw, clean);
-
-    t.equals(raw['boundary.circle.radius'], defaults['boundary:circle:radius:coarse'], 'should be from defaults');
-    t.equals(clean['boundary.circle.radius'], parseFloat(defaults['boundary:circle:radius:coarse']), 'should be same as raw');
     t.end();
-  });
-
-  test('no boundary.circle.radius and coarse layer supplied should be set to coarse default', function(t) {
-    var raw = {
-      'point.lat': '12.121212',
-      'point.lon': '21.212121'
-    };
-    var clean = { layers: 'locality' };
-    var errorsAndWarnings = sanitize(raw, clean);
 
-    t.equals(raw['boundary.circle.radius'], defaults['boundary:circle:radius:coarse'], 'should be from defaults');
-    t.equals(clean['boundary.circle.radius'], parseFloat(defaults['boundary:circle:radius:coarse']), 'should be same as raw');
-    t.end();
   });
 
-  test('explicit boundary.circle.radius should be used instead of default', function(t) {
-    var raw = {
-      'point.lat': '12.121212',
-      'point.lon': '21.212121',
-      'boundary.circle.radius': '3248732857km' // this will never be the default
-    };
-    var clean = {};
-    var errorsAndWarnings = sanitize(raw, clean);
-
-    t.equals(raw['boundary.circle.radius'], '3248732857km', 'should be parsed float');
-    t.equals(clean['boundary.circle.radius'], 3248732857.0, 'should be copied from raw');
-    t.end();
-  });
 };
 
-module.exports.all = function (tape, common) {
+module.exports.all = (tape, common) => {
   function test(name, testFunction) {
-    return tape('SANTIZE _geo_reverse ' + name, testFunction);
+    return tape(`SANTIZE _geo_reverse ${name}`, testFunction);
   }
 
-  for( var testCase in module.exports.tests ){
+  for( const testCase in module.exports.tests ){
     module.exports.tests[testCase](test, common);
   }
 };

test/unit/sanitizer/_geonames_deprecation.js

@@ -0,0 +1,82 @@
+const geonames_deprecation = require('../../../sanitizer/_geonames_deprecation');
+
+module.exports.tests = {};
+
+module.exports.tests.no_warnings_or_errors_conditions = (test, common) => {
+  test('undefined sources should add neither warnings nor errors', (t) => {
+    const clean = {};
+
+    const messages = geonames_deprecation(undefined, clean);
+
+    t.deepEquals(clean, {});
+    t.deepEquals(messages, { errors: [], warnings: [] });
+    t.end();
+
+  });
+
+  test('geonames/gn not in sources should add neither warnings nor errors', (t) => {
+    const clean = {
+      sources: ['source 1', 'source 2'],
+    };
+
+    const messages = geonames_deprecation(undefined, clean);
+
+    t.deepEquals(clean.sources, ['source 1', 'source 2']);
+    t.deepEquals(messages, { errors: [], warnings: [] });
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.error_conditions = (test, common) => {
+  test('only geonames in sources should not modify clean.sources and add error message', (t) => {
+    ['gn', 'geonames'].forEach((sources) => {
+      const clean = {
+        sources: [sources]
+      };
+
+      const messages = geonames_deprecation(undefined, clean);
+
+      t.deepEquals(clean.sources, [sources]);
+      t.deepEquals(messages.errors, ['/reverse does not support geonames']);
+      t.deepEquals(messages.warnings, []);
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.warning_conditions = (test, common) => {
+  test('only geonames in sources should not modify clean.sources and add error message', (t) => {
+    ['gn', 'geonames'].forEach((sources) => {
+      const clean = {
+        sources: ['another source', sources, 'yet another source']
+      };
+
+      const messages = geonames_deprecation(undefined, clean);
+
+      t.deepEquals(clean.sources, ['another source', 'yet another source']);
+      t.deepEquals(messages.errors, []);
+      t.deepEquals(messages.warnings, ['/reverse does not support geonames']);
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SANTIZE _geonames_deprecation ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitizer/_geonames_warnings.js

@@ -0,0 +1,111 @@
+const _ = require('lodash');
+
+const geonames_warnings = require('../../../sanitizer/_geonames_warnings');
+
+const nonAdminProperties = ['number', 'street', 'query', 'category'];
+const adminProperties = ['neighbourhood', 'borough', 'city', 'county', 'state', 'postalcode', 'country'];
+
+module.exports.tests = {};
+
+module.exports.tests.no_errors = (test, common) => {
+  test('undefined clean.parsed_text should exit early', (t) => {
+    const clean = {
+      sources: ['geonames'],
+    };
+
+    const messages = geonames_warnings(undefined, clean);
+
+    t.deepEquals(messages, { errors: [], warnings: [] });
+    t.end();
+
+  });
+
+  test('any non-admin analysis field with only geonames sources should exit early', (t) => {
+    adminProperties.forEach((adminProperty) => {
+      nonAdminProperties.forEach((nonAdminProperty) => {
+        const clean = {
+          sources: ['geonames'],
+          parsed_text: {}
+        };
+        clean.parsed_text[nonAdminProperty] = `${nonAdminProperty} value`;
+        clean.parsed_text[adminProperty] = `${adminProperty} value`;
+
+        const messages = geonames_warnings(undefined, clean);
+
+        t.deepEquals(messages, { errors: [], warnings: [] });
+
+      });
+    });
+    t.end();
+
+  });
+
+  test('any non-admin analysis field with non-geonames sources should exit early', (t) => {
+    adminProperties.forEach((adminProperty) => {
+      nonAdminProperties.forEach((nonAdminProperty) => {
+        const clean = {
+          sources: ['this is not geonames'],
+          parsed_text: {}
+        };
+        clean.parsed_text[nonAdminProperty] = `${nonAdminProperty} value`;
+        clean.parsed_text[adminProperty] = `${adminProperty} value`;
+
+        const messages = geonames_warnings(undefined, clean);
+
+        t.deepEquals(messages, { errors: [], warnings: [] });
+
+      });
+    });
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.error_conditions = (test, common) => {
+  test('any admin analysis field and only geonames sources should return error', (t) => {
+    adminProperties.forEach((property) => {
+      const clean = _.set({ sources: ['geonames'] },
+        ['parsed_text', property], `${property} value`);
+
+      const messages = geonames_warnings(undefined, clean);
+
+      t.deepEquals(messages.errors, ['input contains only administrative area data, ' +
+        'no results will be returned when sources=geonames']);
+      t.deepEquals(messages.warnings, []);
+
+    });
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.warning_conditions = (test, common) => {
+  test('any admin analysis field and only geonames sources should return warning', (t) => {
+    adminProperties.forEach((property) => {
+      const clean = _.set({ sources: ['source 1', 'geonames', 'source 2'] },
+        ['parsed_text', property], `${property} value`);
+
+      const messages = geonames_warnings(undefined, clean);
+
+      t.deepEquals(messages.errors, []);
+      t.deepEquals(messages.warnings, ['input contains only administrative area data, ' +
+        'geonames results will not be returned']);
+
+    });
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SANTIZE _geonames_warnings ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitizer/nearby.js

@@ -31,7 +31,8 @@ module.exports.tests.interface = function(test, common) {
 module.exports.tests.sanitizers = function(test, common) {
   test('check sanitizer list', function (t) {
     var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
-      'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country', 'categories'];
+      'sources', 'sources_and_layers', 'geonames_deprecation', 'size', 'private',
+      'geo_reverse', 'boundary_country', 'categories'];
     t.deepEqual(Object.keys(nearby.sanitizer_list), expected);
     t.end();
   });

test/unit/sanitizer/reverse.js

@@ -37,7 +37,8 @@ module.exports.tests.interface = function(test, common) {
 module.exports.tests.sanitizers = function(test, common) {
   test('check sanitizer list', function (t) {
     var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
-      'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country'];
+      'sources', 'sources_and_layers', 'geonames_deprecation', 'size', 'private',
+      'geo_reverse', 'boundary_country'];
     t.deepEqual(Object.keys(reverse.sanitizer_list), expected);
     t.end();
   });

test/unit/sanitizer/search.js

@@ -1,37 +1,38 @@
-var proxyquire =  require('proxyquire').noCallThru();
+const proxyquire =  require('proxyquire').noCallThru();
+const _ = require('lodash');
 
 module.exports.tests = {};
 
-module.exports.tests.sanitize = function(test, common) {
+module.exports.tests.sanitize = (test, common) => {
-  test('verify that all sanitizers were called as expected', function(t) {
+  test('verify that all sanitizers were called as expected', (t) => {
-    var called_sanitizers = [];
+    const called_sanitizers = [];
 
     // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
     //  were all called correctly
-    var search = proxyquire('../../../sanitizer/search', {
+    const search = proxyquire('../../../sanitizer/search', {
-      '../sanitizer/_deprecate_quattroshapes': function() {
+      '../sanitizer/_deprecate_quattroshapes': () => {
         called_sanitizers.push('_deprecate_quattroshapes');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_single_scalar_parameters': function() {
+      '../sanitizer/_single_scalar_parameters': () => {
         called_sanitizers.push('_single_scalar_parameters');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_text': function() {
+      '../sanitizer/_text': () => {
         called_sanitizers.push('_text');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_iso2_to_iso3': function() {
+      '../sanitizer/_iso2_to_iso3': () => {
         called_sanitizers.push('_iso2_to_iso3');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_city_name_standardizer': function() {
+      '../sanitizer/_city_name_standardizer': () => {
         called_sanitizers.push('_city_name_standardizer');
         return { errors: [], warnings: [] };
       },
       '../sanitizer/_size': function() {
-        if (arguments.length === 0) {
+        if (_.isEmpty(arguments)) {
-          return function() {
+          return () => {
             called_sanitizers.push('_size');
             return { errors: [], warnings: [] };
           };
@@ -41,10 +42,10 @@ module.exports.tests.sanitize = function(test, common) {
         }
 
       },
-      '../sanitizer/_targets': function(type) {
+      '../sanitizer/_targets': (type) => {
         if (['layers', 'sources'].indexOf(type) !== -1) {
-          return function() {
+          return () => {
-            called_sanitizers.push('_targets/' + type);
+            called_sanitizers.push(`_targets/${type}`);
             return { errors: [], warnings: [] };
           };
 
@@ -54,13 +55,13 @@ module.exports.tests.sanitize = function(test, common) {
         }
 
       },
-      '../sanitizer/_sources_and_layers': function() {
+      '../sanitizer/_sources_and_layers': () => {
         called_sanitizers.push('_sources_and_layers');
         return { errors: [], warnings: [] };
       },
       '../sanitizer/_flag_bool': function() {
         if (arguments[0] === 'private' && arguments[1] === false) {
-          return function() {
+          return () => {
             called_sanitizers.push('_flag_bool');
             return { errors: [], warnings: [] };
           };
@@ -71,21 +72,25 @@ module.exports.tests.sanitize = function(test, common) {
         }
 
       },
-      '../sanitizer/_geo_search': function() {
+      '../sanitizer/_geo_search': () => {
         called_sanitizers.push('_geo_search');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_boundary_country': function() {
+      '../sanitizer/_boundary_country': () => {
         called_sanitizers.push('_boundary_country');
         return { errors: [], warnings: [] };
       },
-      '../sanitizer/_categories': function() {
+      '../sanitizer/_categories': () => {
         called_sanitizers.push('_categories');
         return { errors: [], warnings: [] };
       },
+        '../sanitizer/_geonames_warnings': () => {
+        called_sanitizers.push('_geonames_warnings');
+        return { errors: [], warnings: [] };
+      }
     });
 
-    var expected_sanitizers = [
+    const expected_sanitizers = [
       '_single_scalar_parameters',
       '_deprecate_quattroshapes',
       '_text',
@@ -98,26 +103,27 @@ module.exports.tests.sanitize = function(test, common) {
       '_flag_bool',
       '_geo_search',
       '_boundary_country',
-      '_categories'
+      '_categories',
+      '_geonames_warnings'
     ];
 
-    var req = {};
+    const req = {};
-    var res = {};
+    const res = {};
 
-    search.middleware(req, res, function(){
+    search.middleware(req, res, () => {
       t.deepEquals(called_sanitizers, expected_sanitizers);
       t.end();
     });
   });
 };
 
-module.exports.all = function (tape, common) {
+module.exports.all = (tape, common) => {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /search ' + name, testFunction);
+    return tape(`SANTIZE /search ${name}`, testFunction);
   }
 
-  for( var testCase in module.exports.tests ){
+  for( const testCase in module.exports.tests ){
     module.exports.tests[testCase](test, common);
   }
 };

test/unit/schema.js

@@ -3,14 +3,6 @@
 const Joi = require('joi');
 const schema = require('../../schema');
 
-function validate(config) {
-  Joi.validate(config, schema, (err, value) => {
-    if (err) {
-      throw new Error(err.details[0].message);
-    }
-  });
-}
-
 module.exports.tests = {};
 
 module.exports.tests.completely_valid = (test, common) => {
@@ -26,19 +18,29 @@ module.exports.tests.completely_valid = (test, common) => {
         localization: {
           flipNumberAndStreetCountries: ['ABC', 'DEF']
         },
-        requestRetries: 19
+        requestRetries: 19,
+        services: {
+          pip: {
+            url: 'http://locahost'
+          },
+          placeholder: {
+            url: 'http://locahost'
+          }
+        }
       },
       esclient: {
         requestTimeout: 17
       }
     };
 
-    t.doesNotThrow(validate.bind(config));
+    const result = Joi.validate(config, schema);
+
+    t.notOk(result.error);
     t.end();
 
   });
 
-  test('basic valid configuration should not throw error', (t) => {
+  test('basic valid configuration should not throw error and have defaults set', (t) => {
     var config = {
       api: {
         version: 'version value',
@@ -50,7 +52,10 @@ module.exports.tests.completely_valid = (test, common) => {
       }
     };
 
-    t.doesNotThrow(validate.bind(config));
+    const result = Joi.validate(config, schema);
+
+    t.notOk(result.error);
+    t.deepEquals(result.value.api.services, {}, 'missing api.services should default to empty object');
     t.end();
 
   });
@@ -63,7 +68,10 @@ module.exports.tests.api_validation = (test, common) => {
       esclient: {}
     };
 
-    t.throws(validate.bind(null, config), /"api" is required/, 'api should exist');
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"api" is required');
     t.end();
 
   });
@@ -79,7 +87,9 @@ module.exports.tests.api_validation = (test, common) => {
       esclient: {}
     };
 
-    t.doesNotThrow(validate.bind(null, config), 'unknown properties should be allowed');
+    const result = Joi.validate(config, schema);
+
+    t.notOk(result.error);
     t.end();
 
   });
@@ -95,7 +105,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"version" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"version" must be a string');
 
     });
 
@@ -114,7 +127,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"indexName" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"indexName" must be a string');
 
     });
 
@@ -133,7 +149,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"host" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"host" must be a string');
 
     });
 
@@ -153,7 +172,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"legacyUrl" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"legacyUrl" must be a string');
 
     });
 
@@ -173,7 +195,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"accessLog" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"accessLog" must be a string');
 
     });
 
@@ -193,7 +218,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"relativeScores" must be a boolean/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"relativeScores" must be a boolean');
 
     });
 
@@ -213,7 +241,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"localization" must be an object/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"localization" must be an object');
 
     });
 
@@ -234,7 +265,10 @@ module.exports.tests.api_validation = (test, common) => {
       esclient: {}
     };
 
-    t.throws(validate.bind(null, config), /"unknown_property" is not allowed/);
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
 
     t.end();
 
@@ -254,9 +288,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(
+      const result = Joi.validate(config, schema);
-        validate.bind(null, config),
+
-        /"flipNumberAndStreetCountries" must be an array/);
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"flipNumberAndStreetCountries" must be an array');
 
     });
 
@@ -278,7 +313,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"0" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"0" must be a string');
 
     });
 
@@ -300,7 +338,11 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /fails to match the required pattern/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, `"0" with value "${value}" fails to match the required pattern: /^[A-Z]{3}$/`);
+
     });
 
     t.end();
@@ -319,9 +361,10 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(
+      const result = Joi.validate(config, schema);
-        validate.bind(null, config),
+
-        /"requestRetries" must be a number/, 'api.requestRetries should be a number');
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"requestRetries" must be a number');
 
     });
 
@@ -340,10 +383,10 @@ module.exports.tests.api_validation = (test, common) => {
       esclient: {}
     };
 
-    t.throws(
+    const result = Joi.validate(config, schema);
-      validate.bind(null, config),
-      /"requestRetries" must be an integer/, 'api.requestRetries should be an integer');
 
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"requestRetries" must be an integer');
     t.end();
 
   });
@@ -359,27 +402,31 @@ module.exports.tests.api_validation = (test, common) => {
       esclient: {}
     };
 
-    t.throws(
+    const result = Joi.validate(config, schema);
-      validate.bind(null, config),
-      /"requestRetries" must be larger than or equal to 0/, 'api.requestRetries must be positive');
 
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"requestRetries" must be larger than or equal to 0');
     t.end();
 
   });
 
-  test('non-string api.pipService should throw error', (t) => {
+  // api.placeholderService has been moved to api.services.placeholder.url
-    [null, 17, {}, [], true].forEach((value) => {
+  test('any api.placeholderService value should be disallowed', (t) => {
+    [null, 17, {}, [], true, 'http://localhost'].forEach((value) => {
       var config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-          pipService: value
+          placeholderService: value
         },
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"pipService" must be a string/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"placeholderService" is not allowed');
 
     });
 
@@ -387,8 +434,9 @@ module.exports.tests.api_validation = (test, common) => {
 
   });
 
-  test('non-URI-formatted api.pipService should throw error', (t) => {
+  // api.pipService has been moved to api.services.pip.url
-    ['this is not a URI'].forEach((value) => {
+  test('any api.pipService value should be allowed', (t) => {
+    [null, 17, {}, [], true, 'http://localhost'].forEach((value) => {
       var config = {
         api: {
           version: 'version value',
@@ -399,27 +447,135 @@ module.exports.tests.api_validation = (test, common) => {
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"pipService" must be a valid uri/);
+      const result = Joi.validate(config, schema);
+
+      t.notOk(result.error);
+
+    });
+
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.api_services_validation = (test, common) => {
+  test('unsupported children of api.services should be disallowed', (t) => {
+    var config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          unknown_property: 'value'
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
+    t.end();
 
   });
 
+};
+
+module.exports.tests.placeholder_service_validation = (test, common) => {
+  test('timeout and retries not specified should default to 250 and 3', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          placeholder: {
+            url: 'http://localhost'
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.value.api.services.placeholder.timeout, 250);
+    t.equals(result.value.api.services.placeholder.retries, 3);
     t.end();
 
   });
 
-  test('non-http/https api.pipService should throw error', (t) => {
+  test('when api.services.placeholder is defined, url is required', (t) => {
+    var config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          placeholder: {
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"url" is required');
+    t.end();
+
+  });
+
+  test('non-string api.services.placeholder.url should throw error', (t) => {
+    [null, 17, {}, [], true].forEach((value) => {
+      var config = {
+        api: {
+          version: 'version value',
+          indexName: 'index name value',
+          host: 'host value',
+          services: {
+            placeholder: {
+              url: value
+            }
+          }
+        },
+        esclient: {}
+      };
+
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"url" must be a string');
+
+    });
+
+    t.end();
+
+  });
+
+  test('non-http/https api.services.placeholder.url should throw error', (t) => {
     ['ftp', 'git', 'unknown'].forEach((scheme) => {
       var config = {
         api: {
           version: 'version value',
           indexName: 'index name value',
           host: 'host value',
-          pipService: `${scheme}://localhost`
+          services: {
+            placeholder: {
+              url: `${scheme}://localhost`
+            }
+          }
         },
         esclient: {}
       };
 
-      t.throws(validate.bind(null, config), /"pipService" must be a valid uri/);
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"url" must be a valid uri with a scheme matching the https\? pattern');
 
     });
 
@@ -427,22 +583,304 @@ module.exports.tests.api_validation = (test, common) => {
 
   });
 
-  test('http/https api.pipService should not throw error', (t) => {
+  test('non-url children of api.services.placeholder should be disallowed', (t) => {
-    ['http', 'https'].forEach((scheme) => {
     var config = {
       api: {
         version: 'version value',
         indexName: 'index name value',
         host: 'host value',
-          pipService: `${scheme}://localhost`
+        services: {
+          placeholder: {
+            url: 'http://localhost',
+            unknown_property: 'value'
+          }
+        }
       },
       esclient: {}
     };
 
-      t.doesNotThrow(validate.bind(null, config), `${scheme} should be allowed`);
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
+    t.end();
 
   });
 
+};
+
+module.exports.tests.pip_service_validation = (test, common) => {
+  test('timeout and retries not specified should default to 250 and 3', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost'
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.value.api.services.pip.timeout, 250);
+    t.equals(result.value.api.services.pip.retries, 3);
+    t.end();
+
+  });
+
+  test('when api.services.pip is defined, url is required', (t) => {
+    var config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"url" is required');
+    t.end();
+
+  });
+
+  test('non-string api.services.pip.url should throw error', (t) => {
+    [null, 17, {}, [], true].forEach((value) => {
+      var config = {
+        api: {
+          version: 'version value',
+          indexName: 'index name value',
+          host: 'host value',
+          services: {
+            pip: {
+              url: value
+            }
+          }
+        },
+        esclient: {}
+      };
+
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"url" must be a string');
+
+    });
+
+    t.end();
+
+  });
+
+  test('non-http/https api.services.pip.url should throw error', (t) => {
+    ['ftp', 'git', 'unknown'].forEach((scheme) => {
+      var config = {
+        api: {
+          version: 'version value',
+          indexName: 'index name value',
+          host: 'host value',
+          services: {
+            pip: {
+              url: `${scheme}://localhost`
+            }
+          }
+        },
+        esclient: {}
+      };
+
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"url" must be a valid uri with a scheme matching the https\? pattern');
+
+    });
+
+    t.end();
+
+  });
+
+  test('non-url children of api.services.pip should be disallowed', (t) => {
+    var config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost',
+            unknown_property: 'value'
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"unknown_property" is not allowed');
+    t.end();
+
+  });
+
+  test('non-number timeout should throw error', (t) => {
+    [null, 'string', {}, [], false].forEach((value) => {
+      const config = {
+        api: {
+          version: 'version value',
+          indexName: 'index name value',
+          host: 'host value',
+          services: {
+            pip: {
+              url: 'http://localhost',
+              timeout: value
+            }
+          }
+        },
+        esclient: {}
+      };
+
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"timeout" must be a number');
+
+    });
+
+    t.end();
+
+  });
+
+  test('non-integer timeout should throw error', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost',
+            timeout: 17.3
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"timeout" must be an integer');
+    t.end();
+
+  });
+
+  test('negative timeout should throw error', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost',
+            timeout: -1
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"timeout" must be larger than or equal to 0');
+    t.end();
+
+  });
+
+  test('non-number retries should throw error', (t) => {
+    [null, 'string', {}, [], false].forEach((value) => {
+      const config = {
+        api: {
+          version: 'version value',
+          indexName: 'index name value',
+          host: 'host value',
+          services: {
+            pip: {
+              url: 'http://localhost',
+              retries: value
+            }
+          }
+        },
+        esclient: {}
+      };
+
+      const result = Joi.validate(config, schema);
+
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"retries" must be a number');
+
+    });
+
+    t.end();
+
+  });
+
+  test('non-integer retries should throw error', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost',
+            retries: 17.3
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"retries" must be an integer');
+    t.end();
+
+  });
+
+  test('negative retries should throw error', (t) => {
+    const config = {
+      api: {
+        version: 'version value',
+        indexName: 'index name value',
+        host: 'host value',
+        services: {
+          pip: {
+            url: 'http://localhost',
+            retries: -1
+          }
+        }
+      },
+      esclient: {}
+    };
+
+    const result = Joi.validate(config, schema);
+
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"retries" must be larger than or equal to 0');
     t.end();
 
   });
@@ -459,9 +897,10 @@ module.exports.tests.esclient_validation = (test, common) => {
       }
     };
 
-    t.throws(
+    const result = Joi.validate(config, schema);
-      validate.bind(null, config),
+
-      /"esclient" is required/, 'esclient should exist');
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"esclient" is required');
     t.end();
 
   });
@@ -477,9 +916,10 @@ module.exports.tests.esclient_validation = (test, common) => {
         esclient: value
       };
 
-      t.throws(
+      const result = Joi.validate(config, schema);
-        validate.bind(null, config),
+
-        /"esclient" must be an object/, 'esclient should be an object');
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"esclient" must be an object');
 
     });
 
@@ -500,9 +940,10 @@ module.exports.tests.esclient_validation = (test, common) => {
         }
       };
 
-      t.throws(
+      const result = Joi.validate(config, schema);
-        validate.bind(null, config),
+
-        /"requestTimeout" must be a number/, 'esclient.requestTimeout should be a number');
+      t.equals(result.error.details.length, 1);
+      t.equals(result.error.details[0].message, '"requestTimeout" must be a number');
 
     });
 
@@ -522,10 +963,10 @@ module.exports.tests.esclient_validation = (test, common) => {
       }
     };
 
-    t.throws(
+    const result = Joi.validate(config, schema);
-      validate.bind(null, config),
-    /"requestTimeout" must be an integer/, 'esclient.requestTimeout should be an integer');
 
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"requestTimeout" must be an integer');
     t.end();
 
   });
@@ -542,10 +983,10 @@ module.exports.tests.esclient_validation = (test, common) => {
       }
     };
 
-    t.throws(
+    const result = Joi.validate(config, schema);
-      validate.bind(null, config),
-      /"requestTimeout" must be larger than or equal to 0/, 'esclient.requestTimeout must be positive');
 
+    t.equals(result.error.details.length, 1);
+    t.equals(result.error.details[0].message, '"requestTimeout" must be larger than or equal to 0');
     t.end();
 
   });

test/unit/service/configurations/PlaceHolder.js

@@ -0,0 +1,139 @@
+module.exports.tests = {};
+
+const PlaceHolder = require('../../../../service/configurations/PlaceHolder');
+
+module.exports.tests.all = (test, common) => {
+  test('getName should return \'placeholder\'', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    t.equals(placeholder.getName(), 'placeholder');
+    t.equals(placeholder.getBaseUrl(), 'http://localhost:1234/');
+    t.equals(placeholder.getTimeout(), 17);
+    t.equals(placeholder.getRetries(), 19);
+    t.end();
+
+  });
+
+  test('getUrl should return value passed to constructor', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    t.equals(placeholder.getUrl(), 'http://localhost:1234/parser/search');
+    t.end();
+
+  });
+
+  test('getParameters should return object with text and lang from req', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    const req = {
+      clean: {
+        text: 'text value',
+        lang: {
+          iso6393: 'lang value'
+        }
+      }
+    };
+
+    t.deepEquals(placeholder.getParameters(req), { text: 'text value', lang: 'lang value' });
+    t.end();
+
+  });
+
+  test('getHeaders should return empty object', (t) => {
+    const configBlob = {
+      url: 'base url',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    t.deepEquals(placeholder.getHeaders(), {});
+    t.end();
+
+  });
+
+  test('getParameters should not include lang if req.clean.lang is unavailable', (t) => {
+    const configBlob = {
+      url: 'base url',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    const req = {
+      clean: {
+        text: 'text value'
+      }
+    };
+
+    t.deepEquals(placeholder.getParameters(req), { text: 'text value' });
+    t.end();
+
+  });
+
+  test('getParameters should not include lang if req.clean.lang.iso6393 is unavailable', (t) => {
+    const configBlob = {
+      url: 'base url',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    const req = {
+      clean: {
+        text: 'text value',
+        lang: {}
+      }
+    };
+
+    t.deepEquals(placeholder.getParameters(req), { text: 'text value' });
+    t.end();
+
+  });
+
+  test('baseUrl ending in / should not have double /\'s return by getUrl', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234/',
+      timeout: 17,
+      retries: 19
+    };
+
+    const placeholder = new PlaceHolder(configBlob);
+
+    t.deepEquals(placeholder.getUrl(), 'http://localhost:1234/parser/search');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SERVICE CONFIGURATION /PlaceHolder ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/service/configurations/PointInPolygon.js

@@ -0,0 +1,101 @@
+module.exports.tests = {};
+
+const PointInPolygon = require('../../../../service/configurations/PointInPolygon');
+
+module.exports.tests.all = (test, common) => {
+  test('getName should return \'pip\'', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const pointInPolygon = new PointInPolygon(configBlob);
+
+    t.equals(pointInPolygon.getName(), 'pip');
+    t.equals(pointInPolygon.getBaseUrl(), 'http://localhost:1234/');
+    t.equals(pointInPolygon.getTimeout(), 17);
+    t.equals(pointInPolygon.getRetries(), 19);
+    t.end();
+
+  });
+
+  test('getUrl should return value formatted with point.lon/lat passed to constructor', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234'
+    };
+
+    const pointInPolygon = new PointInPolygon(configBlob);
+
+    const req = {
+      clean: { }
+    };
+
+    req.clean['point.lon'] = 21.212121;
+    req.clean['point.lat'] = 12.121212;
+
+    t.equals(pointInPolygon.getUrl(req), 'http://localhost:1234/21.212121/12.121212');
+    t.end();
+
+  });
+
+  test('getHeaders should return an empty object', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const pointInPolygon = new PointInPolygon(configBlob);
+
+    t.deepEquals(pointInPolygon.getHeaders(), {});
+    t.end();
+
+  });
+
+  test('getParameters should return an empty object', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234',
+      timeout: 17,
+      retries: 19
+    };
+
+    const pointInPolygon = new PointInPolygon(configBlob);
+
+    t.deepEquals(pointInPolygon.getParameters(), {});
+    t.end();
+
+  });
+
+  test('baseUrl ending in / should not have double /\'s return by getUrl', (t) => {
+    const configBlob = {
+      url: 'http://localhost:1234/',
+      timeout: 17,
+      retries: 19
+    };
+
+    const pointInPolygon = new PointInPolygon(configBlob);
+
+    const req = {
+      clean: { }
+    };
+
+    req.clean['point.lon'] = 21.212121;
+    req.clean['point.lat'] = 12.121212;
+
+    t.equals(pointInPolygon.getUrl(req), 'http://localhost:1234/21.212121/12.121212');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = (tape, common) => {
+  function test(name, testFunction) {
+    return tape(`SERVICE CONFIGURATION /PointInPolygon ${name}`, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/service/pointinpolygon.js

@@ -1,323 +0,0 @@
-const proxyquire = require('proxyquire').noCallThru();
-
-const setup = require('../../../service/pointinpolygon');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = (test, common) => {
-  test('valid interface', (t) => {
-    const logger = require('pelias-mock-logger')();
-
-    var service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    });
-
-    t.equal(typeof service, 'function', 'service is a function');
-    t.end();
-  });
-};
-
-module.exports.tests.do_nothing_service = (test, common) => {
-  test('undefined PiP uri should return service that logs fact that PiP service is not available', (t) => {
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })();
-
-    service({ lat: 12.121212, lon: 21.212121 }, false, (err) => {
-      t.deepEquals(logger.getWarnMessages(), [
-        'point-in-polygon service disabled'
-      ]);
-      t.equals(err, 'point-in-polygon service disabled, unable to resolve {"lat":12.121212,"lon":21.212121}');
-      t.end();
-    });
-
-  });
-
-  test('service unavailable message should not output centroid when do_not_track is true', (t) => {
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })();
-
-    service({ lat: 12.121212, lon: 21.212121 }, true, (err) => {
-      t.deepEquals(logger.getWarnMessages(), [
-        'point-in-polygon service disabled'
-      ]);
-      t.equals(err, 'point-in-polygon service disabled, unable to resolve centroid');
-      t.end();
-    });
-
-  });
-
-};
-
-module.exports.tests.success = (test, common) => {
-  test('lat and lon should be passed to server', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lon/:lat', (req, res, next) => {
-      t.equals(req.params.lat, '12.121212');
-      t.equals(req.params.lon, '21.212121');
-
-      res.send('{ "field": "value" }');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    service({ lat: 12.121212, lon: 21.212121}, false, (err, results) => {
-      t.notOk(err);
-      t.deepEquals(results, { field: 'value' });
-
-      t.ok(logger.isInfoMessage(`using point-in-polygon service at http://localhost:${port}`));
-      t.notOk(logger.hasErrorMessages());
-
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('do_not_track=true should pass DNT header', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lon/:lat', (req, res, next) => {
-      t.ok(req.headers.hasOwnProperty('dnt'));
-      t.equals(req.params.lat, '12.121212');
-      t.equals(req.params.lon, '21.212121');
-
-      res.send('{ "field": "value" }');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    service({ lat: 12.121212, lon: 21.212121}, true, (err, results) => {
-      t.notOk(err);
-      t.deepEquals(results, { field: 'value' });
-
-      t.ok(logger.isInfoMessage(`using point-in-polygon service at http://localhost:${port}`));
-      t.notOk(logger.hasErrorMessages());
-
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('do_not_track=false should not pass DNT header', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lon/:lat', (req, res, next) => {
-      t.notOk(req.headers.hasOwnProperty('dnt'));
-      t.equals(req.params.lat, '12.121212');
-      t.equals(req.params.lon, '21.212121');
-
-      res.send('{ "field": "value" }');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    service({ lat: 12.121212, lon: 21.212121}, false, (err, results) => {
-      t.notOk(err);
-      t.deepEquals(results, { field: 'value' });
-
-      t.ok(logger.isInfoMessage(`using point-in-polygon service at http://localhost:${port}`));
-      t.notOk(logger.hasErrorMessages());
-
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-};
-
-module.exports.tests.failure = (test, common) => {
-  test('server returning success but non-JSON body should log error and return no results', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lon/:lat', (req, res, next) => {
-      t.equals(req.params.lat, '12.121212');
-      t.equals(req.params.lon, '21.212121');
-
-      res.send('this is not JSON');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    const expectedErrorMsg = `http://localhost:${port}/21.212121/12.121212 returned ` +
-      `status 200 but with non-JSON response: this is not JSON`;
-
-    service({ lat: 12.121212, lon: 21.212121}, false, (err, results) => {
-      t.equals(err, expectedErrorMsg);
-      t.notOk(results);
-      t.ok(logger.isErrorMessage(expectedErrorMsg));
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('server returning 200 & non-JSON body should log sanitized error and return no results when do_not_track', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lon/:lat', (req, res, next) => {
-      t.equals(req.params.lat, '12.121212');
-      t.equals(req.params.lon, '21.212121');
-
-      res.send('this is not JSON');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    const expectedErrorMsg = `http://localhost:${port}/[removed]/[removed] returned ` +
-      `status 200 but with non-JSON response: this is not JSON`;
-
-    service({ lat: 12.121212, lon: 21.212121}, true, (err, results) => {
-      t.equals(err, expectedErrorMsg);
-      t.notOk(results);
-      t.ok(logger.isErrorMessage(expectedErrorMsg));
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('server returning error should log it and return no results', (t) => {
-    const server = require('express')().listen();
-    const port = server.address().port;
-
-    // immediately close the server so to ensure an error response
-    server.close();
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    service({ lat: 12.121212, lon: 21.212121}, false, (err, results) => {
-      t.equals(err.code, 'ECONNREFUSED');
-      t.notOk(results);
-      t.ok(logger.isErrorMessage(/ECONNREFUSED/), 'there should be a connection refused error message');
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('non-OK status should log error and return no results', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lat/:lon', (req, res, next) => {
-      res.status(400).send('a bad request was made');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    const expectedErrorMsg = `http://localhost:${port}/21.212121/12.121212 returned ` +
-      `status 400: a bad request was made`;
-
-    service({ lat: 12.121212, lon: 21.212121}, false, (err, results) => {
-      t.equals(err, expectedErrorMsg);
-      t.notOk(results);
-      t.ok(logger.isErrorMessage(expectedErrorMsg));
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-  test('non-OK status should log sanitized error and return no results when do_not_track=true', (t) => {
-    const pipServer = require('express')();
-    pipServer.get('/:lat/:lon', (req, res, next) => {
-      res.status(400).send('a bad request was made');
-    });
-
-    const server = pipServer.listen();
-    const port = server.address().port;
-
-    const logger = require('pelias-mock-logger')();
-
-    const service = proxyquire('../../../service/pointinpolygon', {
-      'pelias-logger': logger
-    })(`http://localhost:${port}`);
-
-    const expectedErrorMsg = `http://localhost:${port}/[removed]/[removed] returned ` +
-      `status 400: a bad request was made`;
-
-    service({ lat: 12.121212, lon: 21.212121}, true, (err, results) => {
-      t.equals(err, expectedErrorMsg);
-      t.notOk(results);
-      t.ok(logger.isErrorMessage(expectedErrorMsg));
-      t.end();
-
-      server.close();
-
-    });
-
-  });
-
-};
-
-module.exports.all = (tape, common) => {
-  function test(name, testFunction) {
-    return tape(`SERVICE /pointinpolygon ${name}`, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};