Merge pull request #311 from pelias/master

Merge ALL THE THINGS

controller/place.js

@@ -16,7 +16,16 @@ function setup( backend ){
     var query = req.clean.ids.map( function(id) {
       return {
         _index: 'pelias',
-        _type: id.type,
+        /*
+         * some gids aren't resolvable to a single type (ex: osmnode and osmway
+         * both have source osm and layer venue), so expect an array of
+         * possible values. It's important to use `type` here instead of
+         * `_type`, as the former actually queries against the type, and thus
+         * can accept multiple match values.  `_type`, on the other hand,
+         * simply changes the actual URL of the query sent to Elasticsearch to
+         * contain a type, which obviously can only take a single type.
+         */
+        type: id.types,
         _id: id.id
       };
     });

controller/search.js

@@ -36,6 +36,7 @@ function setup( backend, query ){
       // set response data
       else {
         res.data = docs;
+        res.meta = meta;
       }
 
       next();

helper/geojsonify.js

@@ -1,8 +1,10 @@
 
 var GeoJSON = require('geojson'),
     extent = require('geojson-extent'),
-    outputGenerator = require('./outputGenerator'),
+    labelGenerator = require('./labelGenerator'),
-    logger = require('pelias-logger').get('api');
+    logger = require('pelias-logger').get('api'),
+    type_mapping = require('./type_mapping'),
+    _ = require('lodash');
 
 // Properties to be copied
 var DETAILS_PROPS = [
@@ -22,49 +24,21 @@ var DETAILS_PROPS = [
 ];
 
 
-var SOURCES = {
-  'geoname': 'gn',
-  'osmnode': 'osm',
-  'osmway': 'osm',
-  'admin0': 'qs',
-  'admin1': 'qs',
-  'admin2': 'qs',
-  'neighborhood': 'qs',
-  'locality': 'qs',
-  'local_admin': 'qs',
-  'osmaddress': 'osm',
-  'openaddresses': 'oa'
-};
-
 function lookupSource(src) {
-  return SOURCES.hasOwnProperty(src._type) ? SOURCES[src._type] : src._type;
+  var sources = type_mapping.type_to_source;
+  return sources.hasOwnProperty(src._type) ? sources[src._type] : src._type;
 }
 
+/*
+ * Use the type to layer mapping, except for Geonames, where having a full
+ * Elasticsearch document source allows a more specific layer name to be chosen
+ */
 function lookupLayer(src) {
-  switch(src._type) {
+  if (src._type === 'geoname') {
-    case 'osmnode':
+    if (_.contains(src.category, 'admin')) {
-    case 'osmway':
+      if (_.contains(src.category, 'admin:city')) { return 'locality'; }
-      return 'venue';
+      if (_.contains(src.category, 'admin:admin1')) { return 'region'; }
-    case 'admin0':
+      if (_.contains(src.category, 'admin:admin2')) { return 'county'; }
-      return 'country';
-    case 'admin1':
-      return 'region';
-    case 'admin2':
-      return 'county';
-    case 'neighborhood':
-      return 'neighbourhood';
-    case 'locality':
-      return 'locality';
-    case 'local_admin':
-      return 'localadmin';
-    case 'osmaddress':
-    case 'openaddresses':
-      return 'address';
-    case 'geoname':
-      if (src.category && src.category.indexOf('admin') !== -1) {
-        if (src.category.indexOf('admin:city') !== -1) { return 'locality'; }
-        if (src.category.indexOf('admin:admin1') !== -1) { return 'region'; }
-        if (src.category.indexOf('admin:admin2') !== -1) { return 'county'; }
       return 'neighbourhood'; // this could also be 'local_admin'
     }
 
@@ -72,6 +46,10 @@ function lookupLayer(src) {
     if (src.address) { return 'address'; }
   }
 
+  if (_.contains(type_mapping.types, src._type)) {
+    return type_mapping.type_to_layer[src._type];
+  }
+
   logger.warn('[geojsonify]: could not map _type ', src._type);
 
   return src._type;
@@ -138,7 +116,7 @@ function addDetails(src, dst) {
  * @param {object} dst
  */
 function addLabel(src, dst) {
-  dst.label = outputGenerator(src);
+  dst.label = labelGenerator(src);
 }
 
 /**
@@ -177,6 +155,16 @@ function copyProperties( source, props, dst ) {
   });
 }
 
+/**
+ * Create a gid from a document
+ * @TODO modify all importers to create separate source and layer fields to remove mapping
+ *
+ * @param {object} src
+ */
+function makeGid(src) {
+  return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id;
+}
+
 /**
  * Determine and set place id, type, and source
  *
@@ -185,6 +173,7 @@ function copyProperties( source, props, dst ) {
  */
 function addMetaData(src, dst) {
   dst.id = src._id;
+  dst.gid = makeGid(src);
   dst.layer = lookupLayer(src);
   dst.source = lookupSource(src);
 }

helper/labelGenerator.js

@@ -0,0 +1,35 @@
+
+var _ = require('lodash'),
+    check = require('check-types'),
+    schemas = require('./labelSchema.json');
+
+module.exports = function( record ){
+
+  var labelParts = [ record.name.default ];
+
+  var schema = schemas.default;
+  
+  if (record.country_a && record.country_a.length && schemas[record.country_a]) {
+    schema = schemas[record.country_a];
+  }
+  
+  var buildOutput = function(parts, schemaArr, record) {
+    for (var i=0; i<schemaArr.length; i++) {
+      var fieldValue = record[schemaArr[i]];
+      if (check.unemptyString(fieldValue) && !_.contains(parts, fieldValue)) {
+        parts.push( fieldValue );
+        return parts;
+      }
+    }
+    return parts;
+  };
+
+  for (var key in schema) {
+    labelParts = buildOutput(labelParts, schema[key], record);
+  }
+
+  // de-dupe outputs
+  labelParts = _.unique( labelParts );
+
+  return labelParts.join(', ').trim();
+};

helper/labelSchema.json

@@ -4,7 +4,11 @@
     "regional": ["region_a", "region", "country"]
   },
   "GBR": {
-    "local": ["neighbourhood", "county", "localadmin", "locality"],
+    "local": ["neighbourhood", "county", "localadmin", "locality", "region"],
+    "regional": ["county","country","region"]
+  },
+  "SGP": {
+    "local": ["neighbourhood", "region", "county", "localadmin", "locality"],
     "regional": ["county","country","region"]
   },
   "default": {

helper/outputGenerator.js

@@ -1,37 +0,0 @@
-
-var schemas = require('./outputSchema.json');
-
-module.exports = function( record ){
-
-  var adminParts = [];
-
-  var schema = schemas.default;
-  
-  if (record.country_a && record.country_a.length && schemas[record.country_a]) {
-    schema = schemas[record.country_a];
-  }
-  
-  var buildOutput = function(parts, schemaArr, record) {
-    for (var i=0; i<schemaArr.length; i++) {
-      var rec = record[schemaArr[i]];
-      if (rec && rec.length) {
-        parts.push( rec );
-        return parts;
-      }
-    }
-    return parts;
-  };
-
-  for (var key in schema) {
-    adminParts = buildOutput(adminParts, schema[key], record);  
-  }
-
-  var outputs = [ record.name.default ].concat( adminParts );
-
-  // de-dupe outputs
-  outputs = outputs.filter( function( i, pos ) {
-    return outputs.indexOf( i ) === pos;
-  });
-
-  return outputs.join(', ').trim();
-};

helper/query_parser.js

@@ -1,15 +1,17 @@
 
 var parser     = require('addressit');
 var extend     = require('extend');
-var layers_map = require('../query/layers');
+var type_mapping = require('../helper/type_mapping');
 var delim      = ',';
+var check      = require('check-types');
+var logger     = require('pelias-logger').get('api');
 
 module.exports = {};
 
 module.exports.get_layers = function get_layers(query) {
   if (query.length <= 3 ) {
     // no address parsing required
-    return layers_map.coarse;
+    return type_mapping.layer_with_aliases_to_type.coarse;
   }
 };
 
@@ -61,5 +63,12 @@ module.exports.get_parsed_address = function get_parsed_address(query) {
     }
   });
 
+  // if all we found was regions, ignore it as it is not enough information to make smarter decisions
+  if (Object.keys(parsed_text).length === 1 && !check.undefined(parsed_text.regions))
+  {
+    logger.info('Ignoring address parser output, regions only');
+    return null;
+  }
+
   return parsed_text;
 };

helper/type_mapping.js

@@ -0,0 +1,87 @@
+var extend = require('extend'),
+  _ = require('lodash');
+
+var TYPE_TO_SOURCE = {
+  'geoname': 'gn',
+  'osmnode': 'osm',
+  'osmway': 'osm',
+  'admin0': 'qs',
+  'admin1': 'qs',
+  'admin2': 'qs',
+  'neighborhood': 'qs',
+  'locality': 'qs',
+  'local_admin': 'qs',
+  'osmaddress': 'osm',
+  'openaddresses': 'oa'
+};
+
+/*
+ * This doesn't include alias layers such as coarse
+ */
+var TYPE_TO_LAYER = {
+  'geoname': 'venue',
+  'osmnode': 'venue',
+  'osmway': 'venue',
+  'admin0': 'country',
+  'admin1': 'region',
+  'admin2': 'county',
+  'neighborhood': 'neighbourhood',
+  'locality': 'locality',
+  'local_admin': 'localadmin',
+  'osmaddress': 'address',
+  'openaddresses': 'address'
+};
+
+var SOURCE_TO_TYPE = {
+  'gn'            : ['geoname'],
+  'geonames'      : ['geoname'],
+  'oa'            : ['openaddresses'],
+  'openaddresses' : ['openaddresses'],
+  'qs'            : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'osm'           : ['osmaddress', 'osmnode', 'osmway'],
+  'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
+};
+
+/**
+ * This does not included alias layers, those are built separately
+ */
+var LAYER_TO_TYPE = {
+  'venue': ['geoname','osmnode','osmway'],
+  'address': ['osmaddress','openaddresses', 'geoname'],
+  'country': ['admin0', 'geoname'],
+  'region': ['admin1', 'geoname'],
+  'county': ['admin2', 'geoname'],
+  'locality': ['locality', 'geoname'],
+  'localadmin': ['local_admin'],
+  'neighbourhood': ['neighborhood', 'geoname']
+};
+
+var LAYER_ALIASES = {
+  'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin']
+};
+
+var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE);
+
+/*
+ * derive the list of types, sources, and layers from above mappings
+ */
+var TYPES   = Object.keys(TYPE_TO_SOURCE);
+var SOURCES = Object.keys(SOURCE_TO_TYPE);
+var LAYERS  = Object.keys(LAYER_TO_TYPE);
+
+var sourceAndLayerToType = function sourceAndLayerToType(source, layer) {
+  return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]);
+};
+
+module.exports = {
+  types: TYPES,
+  sources: SOURCES,
+  layers: LAYERS,
+  type_to_source: TYPE_TO_SOURCE,
+  type_to_layer: TYPE_TO_LAYER,
+  source_to_type: SOURCE_TO_TYPE,
+  layer_to_type: LAYER_TO_TYPE,
+  layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE,
+  source_and_layer_to_type: sourceAndLayerToType
+};

helper/types.js

@@ -1,13 +1,5 @@
-var valid_types = require( '../query/types' );
+var type_mapping = require( '../helper/type_mapping' );
-
+var _ = require('lodash');
-/**
- * Calculate the set-style intersection of two arrays
- */
-var intersection = function intersection(set1, set2) {
-  return set2.filter(function(value) {
-    return set1.indexOf(value) !== -1;
-  });
-};
 
 /**
  * Combine all types and determine the unique subset
@@ -24,14 +16,14 @@ module.exports = function calculate_types(clean_types) {
    * perform a set intersection of their specified types
    */
   if (clean_types.from_layers || clean_types.from_sources) {
-    var types = valid_types;
+    var types = type_mapping.types;
 
     if (clean_types.from_layers) {
-      types = intersection(types, clean_types.from_layers);
+      types = _.intersection(types, clean_types.from_layers);
     }
 
     if (clean_types.from_sources) {
-      types = intersection(types, clean_types.from_sources);
+      types = _.intersection(types, clean_types.from_sources);
     }
 
     return types;

middleware/confidenceScore.js

@@ -13,17 +13,21 @@
 
 var stats = require('stats-lite');
 var logger = require('pelias-logger').get('api');
+var check = require('check-types');
 
 var RELATIVE_SCORES = true;
 
 function setup(peliasConfig) {
+  if (check.assigned(peliasConfig)) {
     RELATIVE_SCORES = peliasConfig.hasOwnProperty('relativeScores') ? peliasConfig.relativeScores : true;
+  }
   return computeScores;
 }
 
 function computeScores(req, res, next) {
   // do nothing if no result data set
-  if (!res || !res.data || !res.meta) {
+  if (check.undefined(req.clean) || check.undefined(res) ||
+      check.undefined(res.data) || check.undefined(res.meta)) {
     return next();
   }
 
@@ -71,6 +75,7 @@ function computeConfidenceScore(req, mean, stdev, hit) {
   // TODO: look at categories and location
 
   hit.confidence /= checkCount;
+  hit.confidence = Number((hit.confidence).toFixed(3));
 
   logger.debug('[confidence]:', hit.confidence, hit.name.default);
 
@@ -78,16 +83,16 @@ function computeConfidenceScore(req, mean, stdev, hit) {
 }
 
 function checkForDealBreakers(req, hit) {
-  if (!req.clean.parsed_text) {
+  if (check.undefined(req.clean.parsed_text)) {
     return false;
   }
 
-  if (req.clean.parsed_text.state && req.clean.parsed_text.state !== hit.admin1_abbr) {
+  if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.admin1_abbr) {
     logger.debug('[confidence][deal-breaker]: state !== admin1_abbr');
     return true;
   }
 
-  if (req.clean.parsed_text.postalcode && req.clean.parsed_text.postalcode !== hit.zip) {
+  if (check.assigned(req.clean.parsed_text.postalcode) && req.clean.parsed_text.postalcode !== hit.zip) {
     logger.debug('[confidence][deal-breaker]: postalcode !== zip');
     return true;
   }
@@ -117,7 +122,8 @@ function checkDistanceFromMean(score, mean, stdev) {
  */
 function checkName(text, parsed_text, hit) {
   // parsed_text name should take precedence if available since it's the cleaner name property
-  if (parsed_text && parsed_text.name && hit.name.default.toLowerCase() === parsed_text.name.toLowerCase()) {
+  if (check.assigned(parsed_text) && check.assigned(parsed_text.name) &&
+    hit.name.default.toLowerCase() === parsed_text.name.toLowerCase()) {
     return 1;
   }
 
@@ -139,7 +145,9 @@ function checkName(text, parsed_text, hit) {
  * @returns {number}
  */
 function checkQueryType(text, hit) {
-  if (!!text.number && (!hit.address || (hit.address && !hit.address.number))) {
+  if (check.assigned(text) && check.assigned(text.number) &&
+      (check.undefined(hit.address) ||
+      (check.assigned(hit.address) && check.undefined(hit.address.number)))) {
     return 0;
   }
   return 1;
@@ -156,22 +164,23 @@ function checkQueryType(text, hit) {
 function propMatch(textProp, hitProp, expectEnriched) {
 
   // both missing, but expect to have enriched value in result => BAD
-  if (!textProp && !hitProp && expectEnriched) { return 0; }
+  if (check.undefined(textProp) && check.undefined(hitProp) && check.assigned(expectEnriched)) { return 0; }
 
   // both missing, and no enrichment expected => GOOD
-  if (!textProp && !hitProp) { return 1; }
+  if (check.undefined(textProp) && check.undefined(hitProp)) { return 1; }
 
   // text has it, result doesn't => BAD
-  if (textProp && !hitProp) { return 0; }
+  if (check.assigned(textProp) && check.undefined(hitProp)) { return 0; }
 
   // text missing, result has it, and enrichment is expected => GOOD
-  if (!textProp && hitProp && expectEnriched) { return 1; }
+  if (check.undefined(textProp) && check.assigned(hitProp) && check.assigned(expectEnriched)) { return 1; }
 
   // text missing, result has it, enrichment not desired => 50/50
-  if (!textProp && hitProp) { return 0.5; }
+  if (check.undefined(textProp) && check.assigned(hitProp)) { return 0.5; }
 
   // both present, values match => GREAT
-  if (textProp && hitProp && textProp.toString().toLowerCase() === hitProp.toString().toLowerCase()) { return 1; }
+  if (check.assigned(textProp) && check.assigned(hitProp) &&
+      textProp.toString().toLowerCase() === hitProp.toString().toLowerCase()) { return 1; }
 
   // ¯\_(ツ)_/¯
   return 0.7;
@@ -200,7 +209,7 @@ function checkAddress(text, hit) {
   var checkCount = 5;
   var res = 0;
 
-  if (text && text.number && text.street) {
+  if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) {
     res += propMatch(text.number, (hit.address ? hit.address.number : null), false);
     res += propMatch(text.street, (hit.address ? hit.address.street : null), false);
     res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true);

middleware/distance.js

@@ -2,26 +2,31 @@ var geolib = require('geolib');
 var check = require('check-types');
 
 
-function setup() {
+function setup(prefix) {
 
-  return computeDistances;
+  return function (req, res, next) {
+    var opts = {
+      prefix: prefix || 'point.'
+    };
+    return computeDistances(req, res, next, opts);
+  };
 }
 
-function computeDistances(req, res, next) {
+function computeDistances(req, res, next, opts) {
 
   // do nothing if no result data set
   if (!res || !res.data) {
     return next();
   }
 
-  if (!(check.number(req.clean['point.lat']) &&
+  if (!(check.number(req.clean[opts.prefix + 'lat']) &&
-        check.number(req.clean['point.lon']))) {
+        check.number(req.clean[opts.prefix + 'lon']))) {
     return next();
   }
 
   var point = {
-    latitude: req.clean['point.lat'],
+    latitude: req.clean[opts.prefix + 'lat'],
-    longitude: req.clean['point.lon']
+    longitude: req.clean[opts.prefix + 'lon']
   };
 
   res.data.forEach(function (place) {

package.json

@@ -44,15 +44,15 @@
     "geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "geopipes-elasticsearch-backend": "^0.2.0",
-    "lodash": "^3.10.1",
     "iso3166-1": "^0.2.3",
+    "lodash": "^3.10.1",
     "markdown": "0.5.0",
     "microtime": "1.4.0",
     "morgan": "1.5.2",
     "pelias-config": "^1.0.1",
     "pelias-esclient": "0.0.25",
     "pelias-logger": "^0.0.8",
-    "pelias-query": "1.5.0",
+    "pelias-query": "2.0.0",
     "pelias-schema": "1.0.0",
     "pelias-suggester-pipeline": "2.0.2",
     "stats-lite": "^1.0.3",

public/attribution.md

@@ -2,6 +2,6 @@
 * Geocoding by [Pelias](https://mapzen.com/pelias) from [Mapzen](https://mapzen.com)
 * Data from
    * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/)
+   * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation
    * [Quattroshapes](https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md) under [CC-BY-2.0](https://creativecommons.org/licenses/by/2.0/)
    * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/)
-   * and other sources

query/autocomplete.js

@@ -13,7 +13,8 @@ query.score( peliasQuery.view.ngrams, 'must' );
 
 // scoring boost
 query.score( peliasQuery.view.phrase );
-query.score( peliasQuery.view.focus );
+query.score( peliasQuery.view.focus( peliasQuery.view.ngrams ) );
+query.score( peliasQuery.view.popularity(['admin0','admin1','admin2']) );
 
 // --------------------------------
 

query/layers.js

@@ -1,15 +0,0 @@
-/*
- * Mapping from data layers to type values
- */
-
-module.exports = {
-  'venue': ['geoname','osmnode','osmway'],
-  'address': ['osmaddress','openaddresses'],
-  'country': ['admin0'],
-  'region': ['admin1'],
-  'county': ['admin2'],
-  'locality': ['locality'],
-  'localadmin': ['local_admin'],
-  'neighbourhood': ['neighborhood'],
-  'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin'],
-};

query/search.js

@@ -14,7 +14,8 @@ query.score( peliasQuery.view.ngrams, 'must' );
 
 // scoring boost
 query.score( peliasQuery.view.phrase );
-query.score( peliasQuery.view.focus );
+query.score( peliasQuery.view.focus( peliasQuery.view.phrase ) );
+query.score( peliasQuery.view.popularity(['admin0','admin1','admin2']) );
 
 // address components
 query.score( peliasQuery.view.address('housenumber') );
@@ -84,9 +85,9 @@ function generateQuery( clean ){
       check.number(clean['boundary.rect.min_lon']) &&
       check.number(clean['boundary.rect.max_lon']) ){
     vs.set({
-      'boundary:rect:top': clean['boundary.rect.min_lat'],
+      'boundary:rect:top': clean['boundary.rect.max_lat'],
       'boundary:rect:right': clean['boundary.rect.max_lon'],
-      'boundary:rect:bottom': clean['boundary.rect.max_lat'],
+      'boundary:rect:bottom': clean['boundary.rect.min_lat'],
       'boundary:rect:left': clean['boundary.rect.min_lon']
     });
   }

query/sources.js

@@ -1,14 +0,0 @@
-/*
- * Mapping from data sources to type values
- */
-
-module.exports = {
-  'gn'            : ['geoname'],
-  'geonames'      : ['geoname'],
-  'oa'            : ['openaddresses'],
-  'openaddresses' : ['openaddresses'],
-  'qs'            : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
-  'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
-  'osm'           : ['osmaddress', 'osmnode', 'osmway'],
-  'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
-};

query/text_parser.js

@@ -1,4 +1,5 @@
 
+var logger = require('pelias-logger').get('api');
 var adminFields = require('../helper/adminFields')();
 
 /**
@@ -21,9 +22,9 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
 
   // ?
   else {
-    console.warn( 'chaos monkey asks: what happens now?' );
+    logger.warn( 'chaos monkey asks: what happens now?' );
-    console.log( parsed_text );
+    logger.warn( parsed_text );
-    try{ throw new Error(); } catch(e){ console.error( e.stack ); } // print a stack trace
+    try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
   }
 
   // ==== add parsed matches [address components] ====

query/types.js

@@ -1,16 +0,0 @@
-
-// querable types
-
-module.exports = [
-  'geoname',
-  'osmnode',
-  'osmway',
-  'admin0',
-  'admin1',
-  'admin2',
-  'neighborhood',
-  'locality',
-  'local_admin',
-  'osmaddress',
-  'openaddresses'
-];

routes/v1.js

@@ -58,6 +58,7 @@ function addRoutes(app, peliasConfig) {
       sanitisers.search.middleware,
       middleware.types,
       controllers.search(),
+      postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.renamePlacenames(),
       postProc.geocodeJSON(peliasConfig, base),
@@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) {
       sanitisers.autocomplete.middleware,
       middleware.types,
       controllers.search(null, require('../query/autocomplete')),
+      postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.renamePlacenames(),
       postProc.geocodeJSON(peliasConfig, base),
@@ -76,7 +78,7 @@ function addRoutes(app, peliasConfig) {
       sanitisers.reverse.middleware,
       middleware.types,
       controllers.search(undefined, reverseQuery),
-      postProc.distances(),
+      postProc.distances('point.'),
       // reverse confidence scoring depends on distance from origin
       //  so it must be calculated first
       postProc.confidenceScoresReverse(),

sanitiser/_geo_common.js

@@ -2,7 +2,8 @@
  * helper sanitiser methods for geo parameters
  */
 var util = require('util'),
-    check = require('check-types');
+    check = require('check-types'),
+    _ = require('lodash');
 
 /**
  * Parse and validate rect parameter
@@ -140,13 +141,13 @@ function sanitize_point( key_prefix, clean, raw, point_is_required ) {
  *
  * @param {string} key
  * @param {object} clean
- * @param {string} param
+ * @param {string} rawValue
  * @param {bool} latlon_is_required
  */
-function sanitize_coord( key, clean, param, latlon_is_required ) {
+function sanitize_coord( key, clean, rawValue, latlon_is_required ) {
-  var value = parseFloat( param );
+  var parsedValue = parseFloat( rawValue );
-  if ( !isNaN( value ) ) {
+  if ( _.isFinite( parsedValue ) ) {
-    clean[key] = value;
+    clean[key] = parsedValue;
   }
   else if (latlon_is_required) {
     throw new Error( util.format( 'missing param \'%s\'', key ) );

sanitiser/_geo_reverse.js

@@ -1,8 +1,9 @@
 
 var geo_common = require ('./_geo_common');
+var _ = require('lodash');
+var defaults = require('../query/defaults');
 var LAT_LON_IS_REQUIRED = true,
-    CIRCLE_IS_REQUIRED = false,
+    CIRCLE_IS_REQUIRED = false;
-    CIRCLE_MUST_BE_COMPLETE = false;
 
 // validate inputs, convert types and apply defaults
 module.exports = function sanitize( raw, clean ){
@@ -10,19 +11,31 @@ module.exports = function sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
+  // helper function to determine if raw has a boundary.circle property
+  var hasBoundaryCircleField = function(field) {
+    return raw.hasOwnProperty('boundary.circle.' + field);
+  };
+
+  if (['lat', 'lon', 'radius'].some(hasBoundaryCircleField)) {
+    messages.warnings.push('boundary.circle is currently unsupported');
+  }
+
   try {
+    // first verify that point.lat/point.lon are valid
     geo_common.sanitize_point( 'point', clean, raw, LAT_LON_IS_REQUIRED );
 
-    // this hack is to allow point.lat/point.lon to be used interchanagbly
+    // overwrite boundary.circle.lat/lon with point.lat/lon
-    // with boundary.circle.lat/boundary.circle/lon
-    if( !clean.hasOwnProperty('boundary.circle.lat') && clean.hasOwnProperty('point.lat') ){
     raw['boundary.circle.lat'] = clean['point.lat'];
-    }
-    if( !clean.hasOwnProperty('boundary.circle.lon') && clean.hasOwnProperty('point.lon') ){
     raw['boundary.circle.lon'] = clean['point.lon'];
+
+    // if no radius was passed, set the default
+    if ( _.isUndefined( raw['boundary.circle.radius'] ) ) {
+      raw['boundary.circle.radius'] = defaults['boundary:circle:radius'];
     }
 
+    // santize the boundary.circle
     geo_common.sanitize_circle( 'boundary.circle', clean, raw, CIRCLE_IS_REQUIRED );
+
   }
   catch (err) {
     messages.errors.push( err.message );

sanitiser/_ids.js

@@ -1,19 +1,58 @@
 var _ = require('lodash'),
     check = require('check-types'),
-    types = require('../query/types');
+    type_mapping = require('../helper/type_mapping');
 
 var ID_DELIM = ':';
 
-// validate inputs, convert types and apply defaults
+// validate inputs, convert types and apply defaults id generally looks like
-// id generally looks like 'geoname:4163334' (type:id)
+// 'geonames:venue:4163334' (source:layer:id) so, all three are required
-// so, both type and id are required fields.
 
 var lengthError = 'invalid param \'ids\': length must be >0';
 
 var formatError = function(input) {
-  return 'id `' + input + 'is invalid: must be of the format type:id for ex: \'geoname:4163334\'';
+  return 'id `' + input + ' is invalid: must be of the format source:layer:id for ex: \'geonames:venue:4163334\'';
 };
 
+var targetError = function(target, target_list) {
+  return target + ' is invalid. It must be one of these values - [' + target_list.join(', ') + ']';
+};
+
+function sanitizeId(rawId, messages) {
+  var parts = rawId.split(ID_DELIM);
+
+  if ( parts.length < 3 ) {
+    messages.errors.push( formatError(rawId) );
+    return;
+  }
+
+  var source = parts[0];
+  var layer = parts[1];
+  var id = parts.slice(2).join(ID_DELIM);
+
+  // check if any parts of the gid are empty
+  if (_.contains([source, layer, id], '')) {
+    messages.errors.push( formatError(rawId) );
+    return;
+  }
+
+  if (!_.contains(type_mapping.sources, source)) {
+    messages.errors.push( targetError(source, type_mapping.sources) );
+    return;
+  }
+
+  if (!_.contains(type_mapping.layers, layer)) {
+    messages.errors.push( targetError(layer, type_mapping.layers) );
+    return;
+  }
+
+  var types = type_mapping.source_and_layer_to_type(source, layer);
+
+  return {
+    id: id,
+    types: types
+  };
+}
+
 function sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
@@ -42,25 +81,8 @@ function sanitize( raw, clean ){
   }
 
   // cycle through raw ids and set those which are valid
-  var validIds = rawIds.map( function( rawId ){
+  var validIds = rawIds.map(function(rawId) {
-    var param_index = rawId.indexOf(ID_DELIM);
+    return sanitizeId(rawId, messages);
-    var type = rawId.substring(0, param_index );
-    var id   = rawId.substring(param_index + 1);
-
-    // check id format
-    if(!check.contains(rawId, ID_DELIM) || !check.unemptyString( id ) || !check.unemptyString( type )) {
-      messages.errors.push( formatError(rawId) );
-    }
-    // type text must be one of the types
-    else if( !_.contains( types, type ) ){
-      messages.errors.push( type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']' );
-    }
-    else {
-      return {
-        id: id,
-        type: type
-      };
-    }
   });
 
   if (validIds.every(check.object)) {

sanitiser/_single_scalar_parameters.js

@@ -0,0 +1,23 @@
+
+var _ = require('lodash'),
+    check = require('check-types');
+
+// validate inputs
+function sanitize( raw, clean ){
+  // error & warning messages
+  var messages = { errors: [], warnings: [] };
+
+  Object.keys(raw).forEach(function(key) {
+    if (_.isArray(raw[key])) {
+      messages.errors.push('\'' + key + '\' parameter can only have one value');
+    } else if (_.isObject(raw[key])) {
+      messages.errors.push('\'' + key + '\' parameter must be a scalar');
+    }
+
+  });
+
+  return messages;
+}
+
+// export function
+module.exports = sanitize;

sanitiser/_text.js

@@ -19,7 +19,10 @@ function sanitize( raw, clean ){
     clean.text = raw.text;
 
     // parse text with query parser
-    clean.parsed_text = query_parser.get_parsed_address(clean.text);
+    var parsed_text = query_parser.get_parsed_address(clean.text);
+    if (check.assigned(parsed_text)) {
+      clean.parsed_text = parsed_text;
+    }
 
     // try to set layers from query parser results
     clean.types = clean.layers || {};

sanitiser/autocomplete.js

@@ -1,5 +1,6 @@
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
+      singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
       size: require('../sanitiser/_size'),
       private: require('../sanitiser/_flag_bool')('private', false),

sanitiser/place.js

@@ -1,6 +1,7 @@
 
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
+      singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       ids: require('../sanitiser/_ids'),
       private: require('../sanitiser/_flag_bool')('private', false)
     };

sanitiser/reverse.js

@@ -1,8 +1,10 @@
 
+var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
-      layers: require('../sanitiser/_targets')('layers', require('../query/layers')),
+      singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
-      sources: require('../sanitiser/_targets')('sources', require('../query/sources')),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
       size: require('../sanitiser/_size'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_reverse: require('../sanitiser/_geo_reverse'),

sanitiser/search.js

@@ -1,10 +1,12 @@
+var type_mapping = require('../helper/type_mapping');
 
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
+      singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
       size: require('../sanitiser/_size'),
-      layers: require('../sanitiser/_targets')('layers', require( '../query/layers' )),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
-      sources: require('../sanitiser/_targets')('sources', require( '../query/sources' )),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),
       boundary_country: require('../sanitiser/_boundary_country'),

test/ciao/place/basic_place.coffee

@@ -1,6 +1,6 @@
 
 #> basic place
-path: '/v1/place?ids=geoname:1'
+path: '/v1/place?ids=geonames:venue:1'
 
 #? 200 ok
 response.statusCode.should.be.equal 200
@@ -29,5 +29,5 @@ should.not.exist json.geocoding.errors
 should.not.exist json.geocoding.warnings
 
 #? inputs
-json.geocoding.query['ids'].should.eql [{ id: '1', type: 'geoname' }]
+json.geocoding.query['ids'].should.eql [{ id: '1', types: [ 'geoname' ] }]
 should.not.exist json.geocoding.query['size']

test/ciao/reverse/duplicate_parameter_name.coffee

@@ -0,0 +1,30 @@
+
+#> set size
+path: '/v1/reverse?point.lat=1&point.lon=1&param=value1&param=value2'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'param\' parameter can only have one value' ]

test/ciao/reverse/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: venue,address,country,region,county,locality,localadmin,neighbourhood,coarse' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: venue,address,country,region,county,locality,localadmin,neighbourhood,coarse' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_multiple.coffee

@@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
+json.geocoding.query.types['from_layers'].should.eql ["admin0","geoname","admin1"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]
+json.geocoding.query['type'].should.eql ["admin0","geoname","admin1"]

test/ciao/reverse/layers_single.coffee

@@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
+json.geocoding.query.types['from_layers'].should.eql ["admin0","geoname"]
-json.geocoding.query['type'].should.eql ["admin0"]
+json.geocoding.query['type'].should.eql ["admin0","geoname"]

test/ciao/reverse/non_scalar_parameter.coffee

@@ -0,0 +1,30 @@
+
+#> set size
+path: '/v1/reverse?point.lat=1&point.lon=1&parameter[idx]=value'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

test/ciao/reverse/sources_layers_invalid_combo.coffee

@@ -31,6 +31,6 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses","geoname"]
 json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
 should.not.exist json.geocoding.query['type']

test/ciao/reverse/sources_layers_valid_combo.coffee

@@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses","geoname"]
 json.geocoding.query['type'].should.eql ["openaddresses"]

test/ciao/search/layers_alias_address.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses","geoname"]
-json.geocoding.query['type'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query['type'].should.eql ["osmaddress","openaddresses","geoname"]

test/ciao/search/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: venue,address,country,region,county,locality,localadmin,neighbourhood,coarse' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/search/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: venue,address,country,region,county,locality,localadmin,neighbourhood,coarse' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/search/layers_multiple.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
+json.geocoding.query.types['from_layers'].should.eql ["admin0","geoname","admin1"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]
+json.geocoding.query['type'].should.eql ["admin0","geoname","admin1"]

test/ciao/search/layers_single.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
+json.geocoding.query.types['from_layers'].should.eql ["admin0","geoname"]
-json.geocoding.query['type'].should.eql ["admin0"]
+json.geocoding.query['type'].should.eql ["admin0","geoname"]

test/ciao/search/sources_layers_invalid_combo.coffee

@@ -32,6 +32,6 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses","geoname"]
 json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
 should.not.exist json.geocoding.query['type']

test/ciao/search/sources_layers_valid_combo.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses","geoname"]
 json.geocoding.query['type'].should.eql ["openaddresses"]

test/unit/controller/place.js

@@ -41,7 +41,7 @@ module.exports.tests.functional_success = function(test, common) {
 
   test('functional success', function(t) {
     var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', type: [ 'a' ] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
     var res = {
@@ -57,7 +57,7 @@ module.exports.tests.functional_success = function(test, common) {
         t.deepEqual(json.features, expected, 'values correctly mapped');
       }
     };
-    var req = { clean: { ids: [ {'id' : 123, 'type': 'a' } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, types: [ 'a' ] } ] }, errors: [], warnings: [] };
     var next = function next() {
       t.equal(req.errors.length, 0, 'next was called without error');
       t.end();
@@ -70,10 +70,10 @@ module.exports.tests.functional_success = function(test, common) {
 module.exports.tests.functional_failure = function(test, common) {
   test('functional failure', function(t) {
     var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', type: [ 'b' ] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
-    var req = { clean: { ids: [ {'id' : 123, 'type': 'b' } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, types: [ 'b' ] } ] }, errors: [], warnings: [] };
     var next = function( message ){
       t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler');
       t.end();
@@ -85,7 +85,7 @@ module.exports.tests.functional_failure = function(test, common) {
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('GET /doc ' + name, testFunction);
+    return tape('GET /place ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/controller/search.js

@@ -41,6 +41,35 @@ module.exports.tests.functional_success = function(test, common) {
     }
   }];
 
+  var expectedMeta = {
+    scores: [10, 20]
+  };
+
+  var expectedData = [
+    {
+      _id: 'myid1',
+      _score: 10,
+      _type: 'mytype1',
+      admin0: 'country1',
+      admin1: 'state1',
+      admin2: 'city1',
+      center_point: { lat: 100.1, lon: -50.5 },
+      name: { default: 'test name1' },
+      value: 1
+    },
+    {
+      _id: 'myid2',
+      _score: 20,
+      _type: 'mytype2',
+      admin0: 'country2',
+      admin1: 'state2',
+      admin2: 'city2',
+      center_point: { lat: 100.2, lon: -51.5 },
+      name: { default: 'test name2' },
+      value: 2
+    }
+  ];
+
   test('functional success', function (t) {
     var backend = mockBackend('client/search/ok/1', function (cmd) {
       t.deepEqual(cmd, {
@@ -66,6 +95,8 @@ module.exports.tests.functional_success = function(test, common) {
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function next() {
       t.equal(req.errors.length, 0, 'next was called without error');
+      t.deepEqual(res.meta, expectedMeta, 'meta data was set');
+      t.deepEqual(res.data, expectedData, 'data was set');
       t.end();
     };
     controller(req, res, next);

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -27,11 +27,9 @@ module.exports = {
             'function_score': {
               'query': {
                 'match': {
-                  'phrase.default': {
+                  'name.default': {
-                    'analyzer': 'peliasPhrase',
+                    'analyzer': 'peliasOneEdgeGram',
-                    'type': 'phrase',
                     'boost': 1,
-                    'slop': 2,
                     'query': 'test'
                   }
                 }
@@ -52,6 +50,48 @@ module.exports = {
               'score_mode': 'avg',
               'boost_mode': 'replace'
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -27,11 +27,9 @@ module.exports = {
             'function_score': {
               'query': {
                 'match': {
-                  'phrase.default': {
+                  'name.default': {
-                    'analyzer': 'peliasPhrase',
+                    'analyzer': 'peliasOneEdgeGram',
-                    'type': 'phrase',
                     'boost': 1,
-                    'slop': 2,
                     'query': 'test'
                   }
                 }
@@ -52,6 +50,48 @@ module.exports = {
               'score_mode': 'avg',
               'boost_mode': 'replace'
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/autocomplete_linguistic_only.js

@@ -23,6 +23,48 @@ module.exports = {
                 'slop': 2
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/search_boundary_country.js

@@ -33,6 +33,48 @@ module.exports = {
                 'slop': 2
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/search_full_address.js

@@ -26,6 +26,48 @@ module.exports = {
                 'boost': 1
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           },{
             'match': {
               'address.number': {

test/unit/fixture/search_linguistic_bbox.js

@@ -23,6 +23,48 @@ module.exports = {
                 'slop': 2
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       },
@@ -31,9 +73,9 @@ module.exports = {
           'must': [{
             'geo_bounding_box': {
               'center_point': {
-                'top': 47.47,
+                'top': 11.51,
                 'right': -61.84,
-                'bottom': 11.51,
+                'bottom': 47.47,
                 'left': -103.16
               },
               '_cache': true,

test/unit/fixture/search_linguistic_focus.js

@@ -52,6 +52,48 @@ module.exports = {
               'score_mode': 'avg',
               'boost_mode': 'replace'
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/search_linguistic_focus_bbox.js

@@ -52,6 +52,48 @@ module.exports = {
               'score_mode': 'avg',
               'boost_mode': 'replace'
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       },
@@ -60,9 +102,9 @@ module.exports = {
           'must': [{
             'geo_bounding_box': {
               'center_point': {
-                'top': 47.47,
+                'top': 11.51,
                 'right': -61.84,
-                'bottom': 11.51,
+                'bottom': 47.47,
                 'left': -103.16
               },
               '_cache': true,

test/unit/fixture/search_linguistic_focus_null_island.js

@@ -52,6 +52,48 @@ module.exports = {
               'score_mode': 'avg',
               'boost_mode': 'replace'
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/search_linguistic_only.js

@@ -23,6 +23,48 @@ module.exports = {
                 'slop': 2
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           }]
         }
       }

test/unit/fixture/search_partial_address.js

@@ -26,6 +26,48 @@ module.exports = {
                 'boost': 1
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           },{
             'match': {
               'admin0': {

test/unit/fixture/search_regions_address.js

@@ -26,6 +26,48 @@ module.exports = {
                 'boost': 1
               }
             }
+          },
+          {
+            'function_score': {
+              'query': {
+                'filtered': {
+                  'filter': {
+                    'exists': {
+                      'field': 'popularity'
+                    }
+                  }
+                }
+              },
+              'max_boost': 2,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'filter': {
+                'or': [
+                  {
+                    'type': {
+                      'value': 'admin0'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin1'
+                    }
+                  },
+                  {
+                    'type': {
+                      'value': 'admin2'
+                    }
+                  }
+                ]
+              },
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'sqrt',
+                  'field': 'popularity'
+                },
+                'weight': 1
+              }]
+            }
           },{
             'match': {
               'address.number': {

test/unit/helper/geojsonify.js

@@ -141,6 +141,7 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id1',
+          'gid': 'type1:type1:id1',
           'layer': 'type1',
           'source': 'type1',
           'label': '\'Round Midnight Jazz and Blues Bar, test3, Angel',
@@ -169,6 +170,7 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id2',
+          'gid': 'type2:type2:id2',
           'layer': 'type2',
           'source': 'type2',
           'label': 'Blues Cafe, test3, Smithfield',
@@ -194,6 +196,7 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': '34633854',
+          'gid': 'osm:venue:34633854',
           'layer': 'venue',
           'source': 'osm',
           'label': 'Empire State Building, Manhattan, NY',

test/unit/helper/labelGenerator.js

@@ -0,0 +1,228 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// major USA city
+module.exports.tests.san_francisco = function(test, common) {
+  test('san francisco', function(t) {
+    var doc = {
+      'name': { 'default': 'San Francisco' },
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'California',
+      'region_a': 'CA',
+      'county': 'San Francisco County',
+      'locality': 'San Francisco'
+    };
+    t.equal(generator(doc),'San Francisco, San Francisco County, CA');
+    t.end();
+  });
+};
+
+// USA venue
+module.exports.tests.nyc_office = function(test, common) {
+  test('30 West 26th Street', function(t) {
+    var doc = {
+      'name': { 'default': '30 West 26th Street' },
+      'housenumber': '30',
+      'street': 'West 26th Street',
+      'postalcode': '10010',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'New York',
+      'region_a': 'NY',
+      'county': 'New York County',
+      'localadmin': 'Manhattan',
+      'locality': 'New York',
+      'neighbourhood': 'Flatiron District'
+    };
+    t.equal(generator(doc),'30 West 26th Street, Manhattan, NY');
+    t.end();
+  });
+};
+
+// AUS state
+module.exports.tests.new_south_wales = function(test, common) {
+  test('new south wales', function(t) {
+    var doc = {
+      'name': { 'default': 'New South Wales' },
+      'country_a': 'AUS',
+      'country': 'Australia',
+      'region': 'New South Wales'
+    };
+    t.equal(generator(doc),'New South Wales, Australia');
+    t.end();
+  });
+};
+
+// USA state
+module.exports.tests.california = function(test, common) {
+  test('california', function(t) {
+    var doc = {
+      'name': { 'default': 'California' },
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'California',
+      'region_a': 'CA'
+    };
+    t.equal(generator(doc),'California, CA');
+    t.end();
+  });
+};
+
+// IND state
+module.exports.tests.west_bengal = function(test, common) {
+  test('west bengal', function(t) {
+    var doc = {
+      'name': { 'default': 'West Bengal' },
+      'country_a': 'IND',
+      'country': 'India',
+      'region': 'West Bengal'
+    };
+    t.equal(generator(doc),'West Bengal, India');
+    t.end();
+  });
+};
+
+// SGP region
+module.exports.tests.north_west_singapore = function(test, common) {
+  test('north west singapore', function(t) {
+    var doc = {
+      'name': { 'default': 'North West' },
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'North West'
+    };
+    t.equal(generator(doc),'North West, Singapore');
+    t.end();
+  });
+};
+
+// IRQ region
+module.exports.tests.arbil = function(test, common) {
+  test('arbil', function(t) {
+    var doc = {
+      'name': { 'default': 'Arbil' },
+      'country_a': 'IRQ',
+      'country': 'Iraq',
+      'region': 'Arbil'
+    };
+    t.equal(generator(doc),'Arbil, Iraq');
+    t.end();
+  });
+};
+
+// ESP city
+module.exports.tests.madrid = function(test, common) {
+  test('madrid', function(t) {
+    var doc = {
+      'name': { 'default': 'Madrid' },
+      'country_a': 'ESP',
+      'country': 'Spain',
+      'region': 'Madrid'
+    };
+    t.equal(generator(doc),'Madrid, Spain');
+    t.end();
+  });
+};
+
+// GBR street address
+module.exports.tests.one_main_street_uk = function(test, common) {
+  test('one main street uk', function(t) {
+    var doc = {
+      'name': { 'default': '1 Main St' },
+      'housenumber': '1',
+      'street': 'Main St',
+      'postalcode': 'BT77 0BG',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Dungannon'
+    };
+    t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom');
+    t.end();
+  });
+};
+
+// GBR venue
+module.exports.tests.hackney_city_farm = function(test, common) {
+  test('hackney city farm', function(t) {
+    var doc = {
+      'name': { 'default': 'Hackney City Farm' },
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Hackney',
+      'county': 'Greater London',
+      'locality': 'London',
+      'neighbourhood': 'Haggerston'
+    };
+    t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London');
+    t.end();
+  });
+};
+
+// DEU street address
+module.exports.tests.one_grolmanstrasse = function(test, common) {
+  test('one grolmanstrasse', function(t) {
+    var doc = {
+      'name': { 'default': '1 Grolmanstraße' },
+      'housenumber': '1',
+      'street': 'Grolmanstraße',
+      'postalcode': '10623',
+      'country_a': 'DEU',
+      'country': 'Germany',
+      'region': 'Berlin',
+      'county': 'Berlin',
+      'locality': 'Berlin',
+      'neighbourhood': 'Halensee'
+    };
+    t.equal(generator(doc),'1 Grolmanstraße, Berlin, Germany');
+    t.end();
+  });
+};
+
+// NZD country
+module.exports.tests.new_zealand = function(test, common) {
+  test('new zealand', function(t) {
+    var doc = {
+      'name': { 'default': 'New Zealand' },
+      'country_a': 'NZL',
+      'country': 'New Zealand'
+    };
+    t.equal(generator(doc),'New Zealand');
+    t.end();
+  });
+};
+
+// SGP venue
+module.exports.tests.singapore_mcdonalds = function(test, common) {
+  test('singapore_mcdonalds', function(t) {
+    var doc = {
+      'name': { 'default': 'McDonald\'s' },
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'Central Singapore',
+      'locality': 'Singapore'
+    };
+    t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelSchema.js

@@ -1,5 +1,5 @@
 
-var schemas = require('../../../helper/outputSchema.json');
+var schemas = require('../../../helper/labelSchema.json');
 var alpha3  = require('../mock/alpha3.json');
 
 module.exports.tests = {};

test/unit/helper/query_parser.js

@@ -1,6 +1,7 @@
-
 var parser = require('../../../helper/query_parser');
-var layers_map = require('../../../query/layers');
+
+var type_mapping = require('../../../helper/type_mapping');
+var layers_map = type_mapping.layer_with_aliases_to_type;
 
 module.exports.tests = {};
 
@@ -13,26 +14,22 @@ module.exports.tests.interface = function(test, common) {
 };
 
 module.exports.tests.split_on_comma = function(test, common) {
-  var queries = ['soho, new york', 'chelsea, london', '123 main, new york'];
+  var queries = [
-  var delim   = ',';
+    { name: 'soho', admin_parts: 'new york' },
+    { name: 'chelsea', admin_parts: 'london' },
+    { name: '123 main', admin_parts: 'new york' }
+  ];
 
-  var testParse = function(query) {
+  queries.forEach(function (query) {
     test('naive parsing ' + query, function(t) {
-      var address = parser.get_parsed_address(query);
+      var address = parser.get_parsed_address(query.name + ', ' + query.admin_parts);
-      var delimIndex = query.indexOf(delim);
-      var name = query.substring(0, delimIndex);
-      var admin_parts = query.substring(delimIndex + 1).trim();
 
       t.equal(typeof address, 'object', 'valid object');
-      t.equal(address.name, name, 'name set correctly to ' + address.name);
+      t.equal(address.name, query.name, 'name set correctly to ' + address.name);
-      t.equal(address.admin_parts, admin_parts, 'admin_parts set correctly to ' + address.admin_parts);
+      t.equal(address.admin_parts, query.admin_parts, 'admin_parts set correctly to ' + address.admin_parts);
       t.end();
     });
-  };
+  });
-
-  for (var key in queries) {
-    testParse( queries[key] );
-  }
 };
 
 module.exports.tests.parse_three_chars_or_less = function(test, common) {
@@ -40,7 +37,8 @@ module.exports.tests.parse_three_chars_or_less = function(test, common) {
   var num_queries   = ['1', '12', '123'];
   var alphanum_q    = ['a1', '1a2', '12c'];
 
-  var testParse = function(query) {
+  var queries = chars_queries.concat(num_queries).concat(alphanum_q);
+  queries.forEach(function(query) {
     test('query length < 3 (' + query + ')', function(t) {
       var address = parser.get_parsed_address(query);
       var target_layer = layers_map.coarse;
@@ -50,111 +48,64 @@ module.exports.tests.parse_three_chars_or_less = function(test, common) {
       t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
       t.end();
     });
-  };
+  });
-
-  var queries = chars_queries.concat(num_queries).concat(alphanum_q);
-  for (var key in queries) {
-    testParse( queries[key] );
-  }
 };
 
-module.exports.tests.parse_one_or_more_tokens = function(test, common) {
+module.exports.tests.parse_one_token = function(test, common) {
-  var one_token_queries = ['hyderbad', 'yugoslavia', 'somethingreallybigbutjustonetokenstill'];
+  test('query with one token', function (t) {
-  var two_tokens_nonum  = ['small town', 'biggg city', 'another empire'];
+    var address = parser.get_parsed_address('yugolsavia');
-  var two_tokens_withnum= ['123 main', 'sixty 1', '123-980 house'];
+    t.equal(address, null, 'nothing address specific detected');
-
+    t.end();
-  // parse address is now always true to fix pelias/api#194
+  });
-  var testParse = function(query, parse_address) {
+  test('query with two tokens, no numbers', function (t) {
-    test('query with one or more tokens (' + query + ')', function(t) {
+    var address = parser.get_parsed_address('small town');
-      var address = parser.get_parsed_address(query);
+    t.equal(address, null, 'nothing address specific detected');
-      var target_layer = layers_map.coarse.concat(layers_map.venue);
+    t.end();
-      var layers = parser.get_layers(query);
+  });
-
+  test('query with two tokens, number first', function (t) {
-      t.equal(typeof address, 'object', 'valid object');
+    var address = parser.get_parsed_address('123 main');
-
+    t.equal(address, null, 'nothing address specific detected');
-      if (parse_address) {
+    t.end();
-        t.deepEqual(address.regions.join(''), query, 'since query contained a number, it went through address parsing');
+  });
-      } else {
+  test('query with two tokens, number second', function (t) {
-        t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
+    var address = parser.get_parsed_address('main 123');
-      }
+    t.equal(address, null, 'nothing address specific detected');
-
+    t.end();
+  });
+  test('query with many tokens', function(t) {
+    var address = parser.get_parsed_address('main particle new york');
+    t.equal(address, null, 'nothing address specific detected');
     t.end();
   });
-  };
-
-  var queries = one_token_queries.concat(two_tokens_nonum);
-  for (var key in queries) {
-    testParse( queries[key], true );
-  }
-  for (key in two_tokens_withnum) {
-    testParse( two_tokens_withnum[key], true );
-  }
 };
 
 module.exports.tests.parse_address = function(test, common) {
-  var addresses_nonum  = [{ non_street: 'main particle', city: 'new york'},
+  test('valid address, house number', function(t) {
-                          { non_street: 'biggg city block' },
+    var query_string = '123 main st new york ny';
-                          { non_street: 'the empire state building' }
-                         ];
-  var address_with_num = [{ number: 123, street: 'main st', city: 'new york', state: 'ny'},
-                          { number: 456, street: 'pine ave', city: 'san francisco', state: 'CA'},
-                          { number: 1980, street: 'house st', city: 'hoboken', state: 'NY'}
-                         ];
-  var address_with_zip = [{ number: 1, street: 'main st', city: 'new york', state: 'ny', zip: 10010},
-                          { number: 4, street: 'ape ave', city: 'san diego', state: 'CA', zip: 98970},
-                          { number: 19, street: 'house dr', city: 'houston', state: 'TX', zip: 79089}
-                         ];
-
-  var testParse = function(query, hasNumber, hasZip) {
-    var testcase = 'parse query with ' + (hasNumber ? 'a house number ': 'no house number ');
-    testcase += 'and ' + (hasZip ? 'a zip ' : 'no zip ');
-
-    test(testcase, function(t) {
-      var query_string = '';
-      for (var k in query) {
-        query_string += ' ' + query[k];
-      }
-
-      // remove leading whitespace
-      query_string = query_string.substring(1);
-
     var address = parser.get_parsed_address(query_string);
 
-      t.equal(typeof address, 'object', 'valid object for the address ('+query_string+')');
+    t.equal(typeof address, 'object', 'valid object for the address');
-
+    t.equal(address.number, 123, 'parsed house number');
-      if (!hasNumber && !hasZip && query.non_street) {
+    t.equal(address.street, 'main st', 'parsed street');
-        t.equal(address.regions.join(''), query_string, 'expected parsing result');
+    t.deepEqual(address.regions, ['new york'], 'parsed city');
-      } else {
+    t.equal(address.state , 'NY', 'parsed state');
-        t.equal(address.regions.join(''), query.city, 'city in regions (' + query.city +')');
-      }
-
-      if ((hasNumber || hasZip) && query.street) {
-        t.equal(typeof address.number, 'number', 'valid house number format (' + address.number + ')');
-        t.equal(address.number, query.number, 'correct house number (' + query.number + ')');
-        t.equal(typeof address.street, 'string', 'valid street name format (' + address.street + ')');
-        t.equal(address.street, query.street, 'correct street name (' + query.street + ')');
-      }
-
-      if (hasZip) {
-        t.equal(typeof address.postalcode, 'number', 'valid zip (' + address.postalcode + ')');
-        t.equal(address.postalcode, query.zip, 'correct postal code (' + query.zip + ')');
-      }
-
     t.end();
   });
-  };
+  test('valid address, zipcode', function(t) {
+    var query_string = '123 main st new york ny 10010';
+    var address = parser.get_parsed_address(query_string);
 
-  for (var key in addresses_nonum) {
+    t.equal(typeof address, 'object', 'valid object for the address');
-    testParse( addresses_nonum[key] );
+    t.equal(address.number, 123, 'parsed house number');
-  }
+    t.equal(address.street, 'main st', 'parsed street');
-  for (key in address_with_num) {
+    t.deepEqual(address.regions, ['new york'], 'parsed city');
-    testParse( address_with_num[key], true );
+    t.equal(address.state , 'NY', 'parsed state');
-  }
+    t.equal(address.postalcode, 10010, 'parsed zip');
-  for (key in address_with_zip) {
+    t.end();
-    testParse( address_with_zip[key], true, true );
+  });
-  }
 };
 
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {

test/unit/helper/type_mapping.js

@@ -0,0 +1,52 @@
+var check = require('check-types');
+var type_mapping = require('../../../helper/type_mapping');
+
+module.exports.tests = {};
+
+module.exports.tests.interfaces = function(test, common) {
+  test('types list', function(t) {
+    t.ok(check.array(type_mapping.types), 'is array');
+    t.ok(check.hasLength(type_mapping.types, 11), 'has correct number of elements');
+    t.end();
+  });
+
+  test('type to source mapping', function(t) {
+    t.ok(check.object(type_mapping.type_to_source), 'is object');
+    t.ok(check.hasLength(Object.keys(type_mapping.type_to_source), 11), 'has correct number of elements');
+    t.end();
+  });
+
+  test('type to layer mapping', function(t) {
+    t.ok(check.object(type_mapping.type_to_layer), 'is object');
+    t.ok(check.hasLength(Object.keys(type_mapping.type_to_layer), 11), 'has correct number of elements');
+    t.end();
+  });
+
+  test('source to type mapping', function(t) {
+    t.ok(check.object(type_mapping.source_to_type), 'is object');
+    t.ok(check.hasLength(Object.keys(type_mapping.source_to_type), 8), 'has correct number of elements');
+    t.end();
+  });
+
+  test('layer to type mapping', function(t) {
+    t.ok(check.object(type_mapping.layer_to_type), 'is object');
+    t.equal(Object.keys(type_mapping.layer_to_type).length, 8, 'has correct number of elements');
+    t.end();
+  });
+
+  test('layer to type mapping (with aliases)', function(t) {
+    t.ok(check.object(type_mapping.layer_with_aliases_to_type), 'is object');
+    t.ok(check.hasLength(Object.keys(type_mapping.layer_with_aliases_to_type), 9), 'has correct number of elements');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('type_mapping: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/middleware/confidenceScore.js

@@ -0,0 +1,82 @@
+var confidenceScore = require('../../../middleware/confidenceScore')();
+
+module.exports.tests = {};
+
+module.exports.tests.confidenceScore = function(test, common) {
+
+  test('empty res and req should not throw exception', function(t) {
+    try {
+      confidenceScore({}, {}, function() {});
+      t.pass('no exception');
+    }
+    catch (e) {
+      t.fail('an exception should not have been thrown');
+    }
+    finally {
+      t.end();
+    }
+
+  });
+
+  test('res.results without parsed_text should not throw exception', function(t) {
+    var req = {};
+    var res = {
+      data: [{
+        name: 'foo'
+      }],
+      meta: [10]
+    };
+
+    try {
+      confidenceScore(req, res, function() {});
+      t.pass('no exception');
+    }
+    catch (e) {
+      t.fail('an exception should not have been thrown');
+      console.log(e.stack);
+    }
+    finally {
+      t.end();
+    }
+
+  });
+
+
+  test('res.results without parsed_text should not throw exception', function(t) {
+    var req = {
+      clean: { text: 'test name1' }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        admin0: 'country1', admin1: 'state1', admin2: 'city1'
+      }, {
+        value: 2,
+        center_point: { lat: 100.2, lon: -51.5 },
+        name: { default: 'test name2' },
+        admin0: 'country2', admin1: 'state2', admin2: 'city2',
+        _score: 20
+      }],
+      meta: {scores: [10]}
+    };
+
+    confidenceScore(req, res, function() {});
+    t.equal(res.data[0].confidence, 0.6, 'score was set');
+    t.end();
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('[middleware] confidenceScore: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/types.js

@@ -1,23 +0,0 @@
-
-var types = require('../../../query/types');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = function(test, common) {
-  test('valid interface', function(t) {
-    t.true(Array.isArray(types), 'valid array');
-    t.equal(types.length, 11, 'valid array');
-    t.end();
-  });
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('types ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/run.js

@@ -17,19 +17,23 @@ var tests = [
   require('./sanitiser/_layers'),
   require('./sanitiser/reverse'),
   require('./sanitiser/place'),
-  require('./query/types'),
   require('./query/search'),
   require('./query/autocomplete'),
   require('./query/reverse'),
   require('./query/defaults'),
   require('./helper/query_parser'),
   require('./helper/geojsonify'),
-  require('./helper/outputSchema'),
+  require('./helper/labelSchema'),
+  require('./helper/labelGenerator'),
   require('./helper/types'),
+  require('./helper/type_mapping'),
   require('./sanitiser/_geo_common'),
   require('./middleware/distance'),
   require('./middleware/confidenceScoreReverse'),
+  require('./middleware/confidenceScore'),
   require('./sanitiser/_size'),
+  require('./sanitiser/_single_scalar_parameters'),
+  require('./sanitiser/_geo_reverse'),
 ];
 
 tests.map(function(t) {

test/unit/sanitiser/_geo_reverse.js

@@ -0,0 +1,114 @@
+var sanitize = require('../../../sanitiser/_geo_reverse');
+var defaults = require('../../../query/defaults');
+
+module.exports.tests = {};
+
+module.exports.tests.sanitize_boundary_country = function(test, common) {
+  test('raw with boundary.circle.lat should add warning about ignored boundary.circle', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121',
+      'boundary.circle.lat': '13.131313'
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    t.equals(clean['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
+    t.deepEquals(errorsAndWarnings, {
+      errors: [],
+      warnings: ['boundary.circle is currently unsupported']
+    }, 'no warnings/errors');
+    t.end();
+  });
+
+  test('raw with boundary.circle.lon should add warning about ignored boundary.circle', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121',
+      'boundary.circle.lon': '31.313131'
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    t.equals(clean['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
+    t.deepEquals(errorsAndWarnings, {
+      errors: [],
+      warnings: ['boundary.circle is currently unsupported']
+    }, 'no warnings/errors');
+    t.end();
+  });
+
+  test('raw with boundary.circle.radius should add warning about ignored boundary.circle', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121',
+      'boundary.circle.radius': '17'
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    // t.equals(clean['boundary.circle.radius'], 12.121212, 'should be set to point.lat')
+    t.deepEquals(errorsAndWarnings, {
+      errors: [],
+      warnings: ['boundary.circle is currently unsupported']
+    }, 'no warnings/errors');
+    t.end();
+  });
+
+  test('boundary.circle.lat/lon should be overridden with point.lat/lon', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121',
+      'boundary.circle.lat': '13.131313',
+      'boundary.circle.lon': '31.313131'
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    t.equals(raw['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
+    t.equals(raw['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
+    t.equals(clean['boundary.circle.lat'], 12.121212, 'should be set to point.lat');
+    t.equals(clean['boundary.circle.lon'], 21.212121, 'should be set to point.lon');
+    t.end();
+  });
+
+  test('no boundary.circle.radius supplied should be set to default', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121'
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    t.equals(raw['boundary.circle.radius'], defaults['boundary:circle:radius'], 'should be from defaults');
+    t.equals(clean['boundary.circle.radius'], parseFloat(defaults['boundary:circle:radius']), 'should be same as raw');
+    t.end();
+
+  });
+
+  test('explicit boundary.circle.radius should be used instead of default', function(t) {
+    var raw = {
+      'point.lat': '12.121212',
+      'point.lon': '21.212121',
+      'boundary.circle.radius': '3248732857km' // this will never be the default
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+
+    t.equals(raw['boundary.circle.radius'], '3248732857km', 'should be parsed float');
+    t.equals(clean['boundary.circle.radius'], 3248732857.0, 'should be copied from raw');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANTIZE _geo_reverse ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/_ids.js

@@ -1,20 +1,16 @@
 var sanitize = require('../../../sanitiser/_ids');
 
 var delimiter = ':';
-var types = require('../../../query/types');
+var type_mapping = require('../../../helper/type_mapping');
 var inputs = {
   valid: [ 'geoname:1', 'osmnode:2', 'admin0:53', 'osmway:44', 'geoname:5' ],
   invalid: [ ':', '', '::', 'geoname:', ':234', 'gibberish:23' ]
 };
 
 var formatError = function(input) {
-  return 'id `' + input + 'is invalid: must be of the format type:id for ex: \'geoname:4163334\'';
+  return 'id `' + input + ' is invalid: must be of the format source:layer:id for ex: \'geonames:venue:4163334\'';
 };
 var lengthError = 'invalid param \'ids\': length must be >0';
-var defaultMissingTypeError = function(input) {
-    var type = input.split(delimiter)[0];
-    return type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']';
-  };
 
 module.exports.tests = {};
 
@@ -74,50 +70,58 @@ module.exports.tests.invalid_ids = function(test, common) {
     t.end();
   });
 
-  test('invalid id: type name invalid', function(t) {
+  test('invalid id: source name invalid', function(t) {
-    var raw = { ids: 'gibberish:23' };
+    var raw = { ids: 'invalidsource:venue:23' };
     var clean = {};
+    var expected_error = 'invalidsource is invalid. It must be one of these values - [' + type_mapping.sources.join(', ') + ']';
 
     var messages = sanitize(raw, clean);
 
-    t.equal(messages.errors[0], defaultMissingTypeError('gibberish:23'), 'format error returned');
+    t.equal(messages.errors[0], expected_error, 'format error returned');
+    t.equal(clean.ids, undefined, 'ids unset in clean object');
+    t.end();
+  });
+
+  test('invalid id: old style 2 part id', function(t) {
+    var raw = { ids: 'geonames:23' };
+    var clean = {};
+
+    var messages = sanitize(raw, clean);
+
+    t.equal(messages.errors[0], formatError('geonames:23'), 'format error returned');
     t.equal(clean.ids, undefined, 'ids unset in clean object');
     t.end();
   });
 };
 
 module.exports.tests.valid_ids = function(test, common) {
-  test('ids: valid input', function(t) {
+  test('ids: valid input (openaddresses)', function(t) {
-    inputs.valid.forEach( function( input ){
+    var raw = { ids: 'openaddresses:address:20' };
-      var input_parts = input.split(delimiter);
-      var expected_clean = { ids: [ { id: input_parts[1], type: input_parts[0] } ]};
-      var raw = {  ids: input };
     var clean = {};
+    var expected_ids = [{
+      id: '20',
+      types: [ 'openaddresses' ]
+    }];
 
     var messages = sanitize( raw, clean );
 
-      t.deepEqual( messages.errors, [], 'no error (' + input + ')' );
+    t.deepEqual( messages.errors, [], ' no errors');
-      t.deepEqual( clean, expected_clean, 'clean set correctly (' + input + ')');
+    t.deepEqual( clean.ids, expected_ids, 'single type value returned');
-    });
     t.end();
   });
 
-  test('ids: valid input with multiple values' , function(t) {
+  test('ids: valid input (osm)', function(t) {
-    var raw = { ids: inputs.valid.join(',') };
+    var raw = { ids: 'osm:venue:500' };
     var clean = {};
-    var expected_clean={
+    var expected_ids = [{
-      ids: [],
+      id: '500',
-    };
+      types: [ 'osmnode', 'osmway' ]
-    // construct the expected id and type for each valid input
+    }];
-    inputs.valid.forEach( function( input ){
-      var input_parts = input.split(delimiter);
-      expected_clean.ids.push({ id: input_parts[1], type: input_parts[0] });
-    });
 
     var messages = sanitize( raw, clean );
 
-    t.deepEqual( messages.errors, [], 'no errors' );
+    t.deepEqual( messages.errors, [], ' no errors');
-    t.deepEqual( clean, expected_clean, 'clean set correctly' );
+    t.deepEqual( clean.ids, expected_ids, 'osm could be two types, but that\'s ok');
     t.end();
   });
 };
@@ -137,10 +141,10 @@ module.exports.tests.array_of_ids = function(test, common) {
 };
 
 module.exports.tests.multiple_ids = function(test, common) {
-  test('duplicate ids', function(t) {
+  test('multiple ids', function(t) {
-    var expected_clean = { ids: [ { id: '1', type: 'geoname' }, { id: '2', type: 'osmnode' } ] };
+    var raw = { ids: 'geonames:venue:1,osm:venue:2' };
-    var raw = { ids: 'geoname:1,osmnode:2' };
     var clean = {};
+    var expected_clean = { ids: [ { id: '1', types: [ 'geoname' ] }, { id: '2', types: [ 'osmnode', 'osmway' ] } ] };
 
     var messages = sanitize( raw, clean);
 
@@ -153,8 +157,8 @@ module.exports.tests.multiple_ids = function(test, common) {
 
 module.exports.tests.de_dupe = function(test, common) {
   test('duplicate ids', function(t) {
-    var expected_clean = { ids: [ { id: '1', type: 'geoname' }, { id: '2', type: 'osmnode' } ]};
+    var expected_clean = { ids: [ { id: '1', types: [ 'geoname' ] }, { id: '2', types: [ 'osmnode', 'osmway' ] } ] };
-    var raw = { ids: 'geoname:1,osmnode:2,geoname:1' };
+    var raw = { ids: 'geonames:venue:1,osm:venue:2,geonames:venue:1' };
     var clean = {};
 
     var messages = sanitize( raw, clean );

test/unit/sanitiser/_layers.js

@@ -1,5 +1,6 @@
+var type_mapping = require('../../../helper/type_mapping');
 
-var sanitize = require('../../../sanitiser/_targets')('layers', require('../../../query/layers'));
+var sanitize = require('../../../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type);
 
 module.exports.tests = {};
 
@@ -42,7 +43,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     t.end();
   });
   test('address (alias) layer', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
+    var address_layers = ['osmaddress','openaddresses','geoname'];
     var raw = { layers: 'address' };
     var clean = {};
 
@@ -75,7 +76,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     t.end();
   });
   test('address alias layer plus regular layers', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
+    var address_layers = ['osmaddress','openaddresses','geoname'];
     var reg_layers   = ['admin0', 'locality'];
 
     var raw = { layers: 'address,country,locality' };

test/unit/sanitiser/_single_scalar_parameters.js

@@ -0,0 +1,60 @@
+var sanitize = require('../../../sanitiser/_single_scalar_parameters');
+
+module.exports.tests = {};
+
+module.exports.tests.single_scalar_parameters = function(test, common) {
+  test('all duplicate parameters should have error messages returned', function(t) {
+    var raw = {
+      arrayParameter1: ['value1', 'value2'],
+      scalarParameter: 'value',
+      arrayParameter2: ['value3']
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+    t.deepEquals(errorsAndWarnings, {
+      errors: [
+        '\'arrayParameter1\' parameter can only have one value',
+        '\'arrayParameter2\' parameter can only have one value',
+      ],
+      warnings: []
+    });
+    t.end();
+  });
+
+  test('object parameters should have error messages returned', function(t) {
+    var raw = {
+      objectParameter1: { key1: 'value1', key2: 'value2'},
+      scalarParameter: 'value',
+      objectParameter2: { }
+    };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+    t.deepEquals(errorsAndWarnings, {
+      errors: [
+        '\'objectParameter1\' parameter must be a scalar',
+        '\'objectParameter2\' parameter must be a scalar'
+      ],
+      warnings: []
+    });
+    t.end();
+  });
+
+  test('request with all scalar parameters should return empty errors', function(t) {
+    var raw = { scalarParameter1: 'value1', scalarParameter2: 2, scalarParameter3: true };
+    var clean = {};
+    var errorsAndWarnings = sanitize(raw, clean);
+    t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] });
+    t.end();
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANTIZE _single_scalar_parameters ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/_source.js

@@ -1,127 +0,0 @@
-var sanitize = require( '../../../sanitiser/_source' );
-
-var success_response = { error: false };
-
-module.exports.tests = {};
-
-module.exports.tests.no_sources = function(test, common) {
-  test('source is not set', function(t) {
-    var req = {
-      query: { },
-      clean: { }
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
-    t.deepEqual(response.errors, [], 'no error returned');
-    t.deepEqual(response.warnings, [], 'no warnings returned');
-    t.end();
-  });
-
-  test('source is empty string', function(t) {
-    var req = {
-      query: {
-        source: ''
-      },
-      clean: { }
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
-    t.deepEqual(response.errors, [], 'no error returned');
-    t.deepEqual(response.warnings, [], 'no warnings returned');
-    t.end();
-  });
-};
-
-module.exports.tests.valid_sources = function(test, common) {
-  test('geonames source', function(t) {
-    var req = {
-      query: {
-        source: 'geonames'
-      },
-      clean: { }
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(req.clean.types, { from_source: ['geoname'] }, 'clean.types should contain from_source entry with geonames');
-    t.deepEqual(response.errors, [], 'no error returned');
-    t.deepEqual(response.warnings, [], 'no warnings returned');
-    t.end();
-  });
-
-  test('openstreetmap source', function(t) {
-    var req = {
-      query: {
-        source: 'openstreetmap'
-      },
-      clean: { }
-    };
-    var expected_types = {
-      from_source: ['osmaddress', 'osmnode', 'osmway']
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(req.clean.types, expected_types, 'clean.types should contain from_source entry with multiple entries for openstreetmap');
-    t.deepEqual(response.errors, [], 'no error returned');
-    t.deepEqual(response.warnings, [], 'no warnings returned');
-    t.end();
-  });
-
-  test('multiple sources', function(t) {
-    var req = {
-      query: {
-        source: 'openstreetmap,openaddresses'
-      },
-      clean: { }
-    };
-    var expected_types = {
-      from_source: ['osmaddress', 'osmnode', 'osmway', 'openaddresses']
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(req.clean.types, expected_types,
-                'clean.types should contain from_source entry with multiple entries for openstreetmap and openadresses');
-    t.deepEqual(response.errors, [], 'no error returned');
-    t.deepEqual(response.warnings, [], 'no warnings returned');
-    t.end();
-  });
-};
-
-module.exports.tests.invalid_sources = function(test, common) {
-  test('geonames source', function(t) {
-    var req = {
-      query: {
-        source: 'notasource'
-      },
-      clean: { }
-    };
-    var expected_response = {
-      errors: [
-        '\'notasource\' is an invalid source parameter. Valid options: geonames,openaddresses,quattroshapes,openstreetmap'
-      ],
-      warnings: []
-    };
-
-    var response = sanitize(req.query, req.clean);
-
-    t.deepEqual(response, expected_response, 'error with message returned');
-    t.deepEqual(req.clean.types, { }, 'clean.types should remain empty');
-    t.end();
-  });
-};
-
-module.exports.all = function (tape, common) {
-  function test(name, testFunction) {
-    return tape('SANTIZE _source ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/sanitiser/_sources.js

@@ -1,4 +1,5 @@
-var sanitize = require( '../../../sanitiser/_targets' )('sources', require('../../../query/sources'));
+var type_mapping = require('../../../helper/type_mapping');
+var sanitize = require( '../../../sanitiser/_targets' )('sources', type_mapping.source_to_type);
 
 var success_messages = { error: false };
 

test/unit/sanitiser/autocomplete.js

@@ -4,7 +4,7 @@ module.exports.tests = {};
 
 module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
-    var expected = ['text', 'size', 'private', 'geo_autocomplete' ];
+    var expected = ['singleScalarParameters', 'text', 'size', 'private', 'geo_autocomplete' ];
     t.deepEqual(Object.keys(autocomplete.sanitiser_list), expected);
     t.end();
   });

test/unit/sanitiser/place.js

@@ -1,7 +1,7 @@
 var place  = require('../../../sanitiser/place'),
     sanitize = place.sanitize,
     middleware = place.middleware,
-    defaultClean = { ids: [ { id: '123', type: 'geoname' } ], private: false };
+    defaultClean = { ids: [ { id: '123', types: [ 'geoname' ] } ], private: false };
 
 // these are the default values you would expect when no input params are specified.
 module.exports.tests = {};
@@ -19,11 +19,19 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
+module.exports.tests.sanitisers = function(test, common) {
+  test('check sanitiser list', function (t) {
+    var expected = ['singleScalarParameters', 'ids', 'private' ];
+    t.deepEqual(Object.keys(place.sanitiser_list), expected);
+    t.end();
+  });
+};
+
 module.exports.tests.sanitize_private = function(test, common) {
   var invalid_values = [null, -1, 123, NaN, 'abc'];
   invalid_values.forEach(function(value) {
     test('invalid private param ' + value, function(t) {
-      var req = { query: { ids:'geoname:123', 'private': value } };
+      var req = { query: { ids:'geonames:venue:123', 'private': value } };
       sanitize(req, function(){
         t.deepEqual( req.errors, [], 'no errors' );
         t.deepEqual( req.warnings, [], 'no warnings' );
@@ -36,7 +44,7 @@ module.exports.tests.sanitize_private = function(test, common) {
   var valid_values = ['true', true, 1];
   valid_values.forEach(function(value) {
     test('valid private param ' + value, function(t) {
-      var req = { query: { ids:'geoname:123', 'private': value } };
+      var req = { query: { ids:'geonames:venue:123', 'private': value } };
       sanitize(req, function(){
         t.deepEqual( req.errors, [], 'no errors' );
         t.deepEqual( req.warnings, [], 'no warnings' );
@@ -49,7 +57,7 @@ module.exports.tests.sanitize_private = function(test, common) {
   var valid_false_values = ['false', false, 0];
   valid_false_values.forEach(function(value) {
     test('test setting false explicitly ' + value, function(t) {
-      var req = { query: { ids:'geoname:123', 'private': value } };
+      var req = { query: { ids:'geonames:venue:123', 'private': value } };
       sanitize(req, function(){
         t.deepEqual( req.errors, [], 'no errors' );
         t.deepEqual( req.warnings, [], 'no warnings' );
@@ -60,7 +68,7 @@ module.exports.tests.sanitize_private = function(test, common) {
   });
 
   test('test default behavior', function(t) {
-    var req = { query: { ids:'geoname:123' } };
+    var req = { query: { ids:'geonames:venue:123' } };
     sanitize(req, function(){
       t.deepEqual( req.errors, [], 'no errors' );
       t.deepEqual( req.warnings, [], 'no warnings' );
@@ -83,7 +91,7 @@ module.exports.tests.invalid_params = function(test, common) {
 
 module.exports.tests.middleware_success = function(test, common) {
   test('middleware success', function(t) {
-    var req = { query: { ids: 'geoname:123' }};
+    var req = { query: { ids: 'geonames:venue:123' }};
     var next = function(){
       t.deepEqual( req.errors, [], 'no errors' );
       t.deepEqual( req.warnings, [], 'no warnings' );

test/unit/sanitiser/reverse.js

@@ -4,11 +4,13 @@
 var reverse  = require('../../../sanitiser/reverse'),
     sanitize = reverse.sanitize,
     middleware = reverse.middleware,
+    defaults = require('../../../query/defaults'),
     defaultError = 'missing param \'lat\'',
     defaultClean =  { 'point.lat': 0,
                       'point.lon': 0,
                       'boundary.circle.lat': 0,
                       'boundary.circle.lon': 0,
+                      'boundary.circle.radius': parseFloat(defaults['boundary:circle:radius']),
                       types: {
                       },
                       size: 10,
@@ -36,7 +38,7 @@ module.exports.tests.interface = function(test, common) {
 
 module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
-    var expected = ['layers', 'sources', 'size', 'private', 'geo_reverse', 'boundary_country'];
+    var expected = ['singleScalarParameters', 'layers', 'sources', 'size', 'private', 'geo_reverse', 'boundary_country'];
     t.deepEqual(Object.keys(reverse.sanitiser_list), expected);
     t.end();
   });

test/unit/sanitiser/search.js

@@ -25,7 +25,7 @@ module.exports.tests.interface = function(test, common) {
 
 module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
-    var expected = ['text', 'size', 'layers', 'sources', 'private', 'geo_search', 'boundary_country' ];
+    var expected = ['singleScalarParameters', 'text', 'size', 'layers', 'sources', 'private', 'geo_search', 'boundary_country' ];
     t.deepEqual(Object.keys(search.sanitiser_list), expected);
     t.end();
   });
@@ -33,7 +33,7 @@ module.exports.tests.sanitisers = function(test, common) {
 
 module.exports.tests.sanitize_invalid_text = function(test, common) {
   test('invalid text', function(t) {
-    var invalid = [ '', 100, null, undefined, new Date() ];
+    var invalid = [ '', 100, null, undefined ];
     invalid.forEach( function( text ){
       var req = { query: { text: text } };
       sanitize(req, function(){

test/unit/service/search.js

@@ -35,16 +35,21 @@ module.exports.tests.functional_success = function(test, common) {
     }
   ];
 
+  var expectedMeta = {
+    scores: [10, 20]
+  };
+
   test('valid ES query', function(t) {
     var backend = mockBackend( 'client/search/ok/1', function( cmd ){
       t.deepEqual(cmd, example_valid_es_query, 'no change to the command');
     });
-    setup( backend, example_valid_es_query, function(err, data) {
+    setup( backend, example_valid_es_query, function(err, data, meta) {
       t.true(Array.isArray(data), 'returns an array');
       data.forEach(function(d) {
         t.true(typeof d === 'object', 'valid object');
       });
       t.deepEqual(data, expected, 'values correctly mapped');
+      t.deepEqual(meta, expectedMeta, 'meta data correctly mapped');
       t.end();
     });
   });