Merge pull request #491 from pelias/staging

Merge staging into production

.travis.yml

@@ -1,6 +1,23 @@
+sudo: false
 language: node_js
-script: "npm run unit"
 node_js:
-  - "0.10"
+  - 0.10
-  - "0.12"
+  - 0.12
-sudo: false
+  - 4.4
+  - 5.8
+matrix:
+    allow_failures:
+        - node_js: 4.4
+        - node_js: 5.8
+env:
+  global:
+    - CXX=g++-4.8
+  matrix:
+    - TEST_SUITE=unit
+script: "npm run $TEST_SUITE"
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8

EXAMPLES.md

@@ -1,9 +0,0 @@
-
-#### Search:
-
-- Nearest Museums: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=museum&size=40
-- Nearest Hotels: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=hotel&size=40
-
-#### Autocomplete:
-
-- Local Neighborhoods: http://pelias.mapzen.com/suggest/nearby?lat=40.7259&lon=-73.9806&input=e&layers=neighborhood&size=40

bin/generate-docs

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+rm -r docs || true
+
+curl -s http://localhost:3100/v1 > /dev/null || die "Pelias server does not appear to be running \
+on http://localhost:3100, run npm start in another window before generating docs."
+
+cd test/ciao
+node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs

controller/place.js

@@ -1,4 +1,5 @@
 var service = { mget: require('../service/mget') };
+var logger = require('pelias-logger').get('api:controller:place');
 
 function setup( backend ){
 
@@ -13,40 +14,18 @@ function setup( backend ){
       return next();
     }
 
-    /* req.clean.ids contains an array of objects with id and types properties.
+    var query = req.clean.ids.map( function(id) {
-     * types is an array of one or more types, since it can't always be known which single
-     * type a gid might belong to (osmnode and osmway both have source osm and layer venue).
-     *
-     * However, the mget Elasticsearch query only accepts a single type at a
-     * time.
-     *
-     * So, first create a new array that, has an entry
-     * with each type and id combination. This requires creating a new array with more entries
-     * than req.clean.ids in the case where entries have multiple types.
-     */
-
-    var recordsToReturn = req.clean.ids.reduce(function (acc, ids_element) {
-      ids_element.types.forEach(function(type) {
-        acc.push({
-          id: ids_element.id,
-          type: type
-        });
-      });
-      return acc;
-    }, []);
-
-    /*
-     * Next, map the list of records to an Elasticsearch mget query
-     */
-    var query = recordsToReturn.map( function(id) {
       return {
         _index: 'pelias',
-        _type: id.type,
+        _type: id.layers,
         _id: id.id
       };
     });
 
+    logger.debug( '[ES req]', JSON.stringify(query) );
+
     service.mget( backend, query, function( err, docs ) {
+      console.log('err:' + err);
 
       // error handler
       if( err ){
@@ -56,6 +35,7 @@ function setup( backend ){
       else {
         res.data = docs;
       }
+      logger.debug('[ES response]', JSON.stringify(docs));
 
       next();
     });

controller/search.js

@@ -8,7 +8,6 @@ function setup( backend, query ){
   query = query || require('../query/search');
 
   function controller( req, res, next ){
-
     // do not run controller when a request
     // validation error has occurred.
     if( req.errors && req.errors.length ){
@@ -25,11 +24,13 @@ function setup( backend, query ){
       body: query( req.clean )
     };
 
-    // ?
+    // use layers field for filtering by type
-    if( req.clean.hasOwnProperty('type') ){
+    if( req.clean.hasOwnProperty('layers') ){
-      cmd.type = req.clean.type;
+      cmd.type = req.clean.layers;
     }
 
+    logger.debug( '[ES req]', JSON.stringify(cmd) );
+
     // query backend
     service.search( backend, cmd, function( err, docs, meta ){
 
@@ -42,7 +43,7 @@ function setup( backend, query ){
         res.data = docs;
         res.meta = meta;
       }
-
+      logger.debug('[ES response]', JSON.stringify(docs));
       next();
     });
 

helper/adminFields.js

@@ -1,45 +0,0 @@
-
-var _ = require('lodash'),
-    peliasSchema = require('pelias-schema'),
-    peliasLogger = require( 'pelias-logger' ).get( 'api' );
-
-var ADMIN_FIELDS = [
-  'admin0',
-  'admin1',
-  'admin1_abbr',
-  'admin2',
-  'local_admin',
-  'locality',
-  'neighborhood',
-  'address.zip'
-];
-
-/**
- * Get all admin fields that were expected and also found in schema
- *
- * @param {Object} [schema] optional: for testing only
- * @param {Array} [expectedFields] optional: for testing only
- * @param {Object} [logger] optional: for testing only
- * @returns {Array.<string>}
- */
-function getAvailableAdminFields(schema, expectedFields, logger) {
-
-  schema = schema || peliasSchema;
-  expectedFields = expectedFields || ADMIN_FIELDS;
-  logger = logger || peliasLogger;
-
-  var actualFields = Object.keys(schema.mappings._default_.properties);
-
-  // check if expected fields are actually in current schema
-  var available = expectedFields.filter(function (field) {
-    return _.contains( actualFields, field );
-  });
-
-  if (available.length === 0) {
-    logger.error('helper/adminFields: no expected admin fields found in schema');
-  }
-
-  return available;
-}
-
-module.exports = getAvailableAdminFields;

helper/geojsonify.js

@@ -4,6 +4,7 @@ var GeoJSON = require('geojson'),
     labelGenerator = require('./labelGenerator'),
     logger = require('pelias-logger').get('api'),
     type_mapping = require('./type_mapping'),
+    Document = require('pelias-model').Document,
     _ = require('lodash');
 
 // Properties to be copied
@@ -11,48 +12,41 @@ var DETAILS_PROPS = [
   'housenumber',
   'street',
   'postalcode',
-  'country_a',
+  'confidence',
+  'distance',
   'country',
+  'country_gid',
+  'country_a',
+  'macroregion',
+  'macroregion_gid',
+  'macroregion_a',
   'region',
+  'region_gid',
   'region_a',
+  'macrocounty',
+  'macrocounty_gid',
+  'macrocounty_a',
   'county',
+  'county_gid',
+  'county_a',
   'localadmin',
+  'localadmin_gid',
+  'localadmin_a',
   'locality',
+  'locality_gid',
+  'locality_a',
   'neighbourhood',
-  'confidence',
+  'neighbourhood_gid',
-  'distance'
+  'bounding_box'
 ];
 
 
 function lookupSource(src) {
-  var sources = type_mapping.type_to_source;
+  return src.source;
-  return sources.hasOwnProperty(src._type) ? sources[src._type] : src._type;
 }
 
-/*
- * Use the type to layer mapping, except for Geonames, where having a full
- * Elasticsearch document source allows a more specific layer name to be chosen
- */
 function lookupLayer(src) {
-  if (src._type === 'geoname') {
+  return src.layer;
-    if (_.contains(src.category, 'admin')) {
-      if (_.contains(src.category, 'admin:city')) { return 'locality'; }
-      if (_.contains(src.category, 'admin:admin1')) { return 'region'; }
-      if (_.contains(src.category, 'admin:admin2')) { return 'county'; }
-      return 'neighbourhood'; // this could also be 'local_admin'
-    }
-
-    if (src.name) { return 'venue'; }
-    if (src.address) { return 'address'; }
-  }
-
-  if (_.contains(type_mapping.types, src._type)) {
-    return type_mapping.type_to_layer[src._type];
-  }
-
-  logger.warn('[geojsonify]: could not map _type ', src._type);
-
-  return src._type;
 }
 
 function geojsonifyPlaces( docs ){
@@ -64,11 +58,20 @@ function geojsonifyPlaces( docs ){
       return !!doc;
     });
 
+  // get all the bounding_box corners as well as single points
+  // to be used for computing the overall bounding_box for the FeatureCollection
+  var extentPoints = extractExtentPoints(geodata);
+
   // convert to geojson
   var geojson             = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] });
+  var geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] });
+
+  // to insert the bbox property at the top level of each feature, it must be done separately after
+  // initial geojson construction is finished
+  addBBoxPerFeature(geojson);
 
   // bounding box calculations
-  computeBBox(geojson);
+  computeBBox(geojson, geojsonExtentPoints);
 
   return geojson;
 }
@@ -116,7 +119,64 @@ function addDetails(src, dst) {
  * @param {object} dst
  */
 function addLabel(src, dst) {
-  dst.label = labelGenerator(src);
+  dst.label = labelGenerator(dst);
+}
+
+/**
+ * Add bounding box
+ *
+ * @param {object} geojson
+ */
+function addBBoxPerFeature(geojson) {
+  geojson.features.forEach(function (feature) {
+
+    if (!feature.properties.hasOwnProperty('bounding_box')) {
+      return;
+    }
+
+    if (feature.properties.bounding_box) {
+      feature.bbox = [
+        feature.properties.bounding_box.min_lon,
+        feature.properties.bounding_box.min_lat,
+        feature.properties.bounding_box.max_lon,
+        feature.properties.bounding_box.max_lat
+      ];
+    }
+
+    delete feature.properties.bounding_box;
+  });
+}
+
+/**
+ * Collect all points from the geodata.
+ * If an item is a single point, just use that.
+ * If an item has a bounding box, add two corners of the box as individual points.
+ *
+ * @param {Array} geodata
+ * @returns {Array}
+ */
+function extractExtentPoints(geodata) {
+  var extentPoints = [];
+  geodata.forEach(function (place) {
+    if (place.bounding_box) {
+      extentPoints.push({
+        lng: place.bounding_box.min_lon,
+        lat: place.bounding_box.min_lat
+      });
+      extentPoints.push({
+        lng: place.bounding_box.max_lon,
+        lat: place.bounding_box.max_lat
+      });
+    }
+    else {
+      extentPoints.push({
+        lng: place.lng,
+        lat: place.lat
+      });
+    }
+  });
+
+  return extentPoints;
 }
 
 /**
@@ -125,17 +185,17 @@ function addLabel(src, dst) {
  *
  * @param {object} geojson
  */
-function computeBBox(geojson) {
+function computeBBox(geojson, geojsonExtentPoints) {
   // @note: extent() sometimes throws Errors for unusual data
   // eg: https://github.com/pelias/pelias/issues/84
   try {
-    var bbox = extent( geojson );
+    var bbox = extent( geojsonExtentPoints );
     if( !!bbox ){
       geojson.bbox = bbox;
     }
   } catch( e ){
     console.error( 'bbox error', e.message, e.stack );
-    console.error( 'geojson', JSON.stringify( geojson, null, 2 ) );
+    console.error( 'geojson', JSON.stringify( geojsonExtentPoints, null, 2 ) );
   }
 }
 
@@ -149,9 +209,24 @@ function computeBBox(geojson) {
  */
 function copyProperties( source, props, dst ) {
   props.forEach( function ( prop ) {
+
     if ( source.hasOwnProperty( prop ) ) {
+
+      // array value, take first item in array (at this time only used for admin values)
+      if (source[prop] instanceof Array) {
+        if (source[prop].length === 0) {
+          return;
+        }
+        if (source[prop][0]) {
+          dst[prop] = source[prop][0];
+        }
+      }
+
+      // simple value
+      else {
         dst[prop] = source[prop];
       }
+    }
   });
 }
 
@@ -162,7 +237,8 @@ function copyProperties( source, props, dst ) {
  * @param {object} src
  */
 function makeGid(src) {
-  return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id;
+  var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
+  return doc.getGid();
 }
 
 /**
@@ -176,6 +252,9 @@ function addMetaData(src, dst) {
   dst.gid = makeGid(src);
   dst.layer = lookupLayer(src);
   dst.source = lookupSource(src);
+  if (src.hasOwnProperty('bounding_box')) {
+    dst.bounding_box = src.bounding_box;
+  }
 }
 
 /**

helper/labelGenerator.js

@@ -1,35 +1,74 @@
-
 var _ = require('lodash'),
-    check = require('check-types'),
+    schemas = require('./labelSchema');
-    schemas = require('./labelSchema.json');
 
 module.exports = function( record ){
+  var schema = getSchema(record.country_a);
 
-  var labelParts = [ record.name.default ];
+  var labelParts = getInitialLabel(record);
 
-  var schema = schemas.default;
+  for (var key in schema) {
+    var valueFunction = schema[key];
 
-  if (record.country_a && record.country_a.length && schemas[record.country_a]) {
+    labelParts = valueFunction(record, labelParts);
-    schema = schemas[record.country_a];
   }
 
-  var buildOutput = function(parts, schemaArr, record) {
+  // NOTE: while it may seem odd to call `uniq` on the list of label parts,
-    for (var i=0; i<schemaArr.length; i++) {
+  //  the effect is quite subtle.  Take, for instance, a result for "Lancaster, PA"
-      var fieldValue = record[schemaArr[i]];
+  //  the pseudo-object is:
-      if (check.unemptyString(fieldValue) && !_.contains(parts, fieldValue)) {
+  //  {
-        parts.push( fieldValue );
+  //    'name': 'Lancaster',
-        return parts;
+  //    'locality': 'Lancaster',
+  //    'region_a': 'PA',
+  //    'country_a': 'USA'
+  //  }
+  //
+  //  the code up to this point generates the label:
+  //  `Lancaster, Lancaster, PA, USA`
+  //
+  //  then the `unique` call reduces this to:
+  //  `Lancaster, PA, USA`
+  //
+  //  this code doesn't have the same effect in the case of a venue or address
+  //  where the `name` field would contain the address or name of a point-of-interest
+  //
+  //  Also see https://github.com/pelias/api/issues/429 for other ways that this is bad
+  //
+  // de-dupe, join, trim
+  return _.uniq( labelParts ).join(', ').trim();
+
+};
+
+function getSchema(country_a) {
+  if (country_a && country_a.length && schemas[country_a]) {
+    return schemas[country_a];
   }
+
+  return schemas.default;
+
 }
-    return parts;
-  };
 
-  for (var key in schema) {
+// helper function that sets a default label for non-US/CA regions
-    labelParts = buildOutput(labelParts, schema[key], record);
+// this is a very special case
+function getInitialLabel(record) {
+  if (isRegion(record.layer) &&
+      isGeonamesOrWhosOnFirst(record.source) &&
+      isUSAOrCAN(record.country_a)) {
+    return [];
   }
 
-  // de-dupe outputs
+  return [record.name];
-  labelParts = _.unique( labelParts );
 
-  return labelParts.join(', ').trim();
+}
-};
+
+function isRegion(layer) {
+  return 'region' === layer;
+}
+
+function isUSAOrCAN(country_a) {
+  return 'USA' === country_a || 'CAN' === country_a;
+}
+
+function isGeonamesOrWhosOnFirst(source) {
+  return 'geonames' === source || 'whosonfirst' === source;
+
+}

helper/labelSchema.js

@@ -0,0 +1,66 @@
+var _ = require('lodash'),
+    check = require('check-types');
+
+module.exports = {
+  'USA': {
+    'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county']),
+    'regional': getUsState,
+    'country': getFirstProperty(['country_a'])
+  },
+  'GBR': {
+    'local': getFirstProperty(['neighbourhood', 'county', 'localadmin', 'locality', 'macroregion', 'region']),
+    'regional': getFirstProperty(['county','country','region'])
+  },
+  'SGP': {
+    'local': getFirstProperty(['neighbourhood', 'region', 'county', 'localadmin', 'locality']),
+    'regional': getFirstProperty(['county','country','region'])
+  },
+  'SWE': {
+    'local': getFirstProperty(['neighbourhood', 'region', 'county', 'localadmin', 'locality']),
+    'regional': getFirstProperty(['country'])
+  },
+  'default': {
+    'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county', 'macroregion', 'region']),
+    'regional': getFirstProperty(['country'])
+  }
+};
+
+// find the first field of record that has a non-empty value that's not already in labelParts
+function getFirstProperty(fields) {
+  return function(record, labelParts) {
+    for (var i = 0; i < fields.length; i++) {
+      var fieldValue = record[fields[i]];
+
+      if (check.nonEmptyString(fieldValue) && !_.includes(labelParts, fieldValue)) {
+        labelParts.push( fieldValue );
+        return labelParts;
+      }
+
+    }
+
+    return labelParts;
+
+  };
+
+}
+
+// this function is exclusively used for figuring out which field to use for US States
+// 1.  if a US state is the most granular bit of info entered, the label should contain
+//  the full state name, eg: Pennsylvania, USA
+// 2.  otherwise, the state abbreviation should be used, eg: Lancaster, PA, USA
+// 3.  if for some reason the abbreviation isn't available, use the full state name
+function getUsState(record, labelParts) {
+  if ('region' === record.layer && record.region) {
+    // add full state name when state is the most granular piece of info
+    labelParts.push(record.region);
+  } else if (record.region_a) {
+    // otherwise just add the region code when available
+    labelParts.push(record.region_a);
+  } else if (record.region) {
+    // add the full name when there's no region code available ()
+    labelParts.push(record.region);
+  }
+
+  return labelParts;
+
+}

helper/labelSchema.json

@@ -1,22 +0,0 @@
-{
-  "USA": {
-    "local": ["localadmin", "locality", "neighbourhood", "county"],
-    "regional": ["region_a", "region", "country"]
-  },
-  "GBR": {
-    "local": ["neighbourhood", "county", "localadmin", "locality", "region"],
-    "regional": ["county","country","region"]
-  },
-  "SGP": {
-    "local": ["neighbourhood", "region", "county", "localadmin", "locality"],
-    "regional": ["county","country","region"]
-  },
-  "SWE": {
-    "local": ["neighbourhood", "region", "county", "localadmin", "locality"],
-    "regional": ["country"]
-  },
-  "default": {
-    "local": ["localadmin", "locality", "neighbourhood", "county", "region"],
-    "regional": ["country"]
-  }
-}

helper/placeTypes.js

@@ -0,0 +1,11 @@
+module.exports = [
+  'country',
+  'macroregion',
+  'region',
+  'macrocounty',
+  'county',
+  'localadmin',
+  'locality',
+  'borough',
+  'neighbourhood'
+];

helper/text_parser.js

@@ -15,7 +15,7 @@ module.exports = {};
 module.exports.get_layers = function get_layers(query) {
   if (query.length <= 3 ) {
     // no address parsing required
-    return type_mapping.layer_with_aliases_to_type.coarse;
+    return type_mapping.layer_mapping.coarse;
   }
 };
 

helper/type_mapping.js

@@ -1,87 +1,81 @@
 var extend = require('extend'),
   _ = require('lodash');
 
-var TYPE_TO_SOURCE = {
+function addStandardTargetsToAliases(standard, aliases) {
-  'geoname': 'gn',
+  var combined = _.extend({}, aliases);
-  'osmnode': 'osm',
+  standard.forEach(function(target) {
-  'osmway': 'osm',
+    if (combined[target] === undefined) {
-  'admin0': 'qs',
+      combined[target] = [target];
-  'admin1': 'qs',
+    }
-  'admin2': 'qs',
+  });
-  'neighborhood': 'qs',
+
-  'locality': 'qs',
+  return combined;
-  'local_admin': 'qs',
+}
-  'osmaddress': 'osm',
-  'openaddresses': 'oa'
-};
 
 /*
- * This doesn't include alias layers such as coarse
+ * Sources
  */
-var TYPE_TO_LAYER = {
-  'geoname': 'venue',
-  'osmnode': 'venue',
-  'osmway': 'venue',
-  'admin0': 'country',
-  'admin1': 'region',
-  'admin2': 'county',
-  'neighborhood': 'neighbourhood',
-  'locality': 'locality',
-  'local_admin': 'localadmin',
-  'osmaddress': 'address',
-  'openaddresses': 'address'
-};
 
-var SOURCE_TO_TYPE = {
+// a list of all sources
-  'gn'            : ['geoname'],
+var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'whosonfirst'];
-  'geonames'      : ['geoname'],
+
+/*
+ * A list of alternate names for sources, mostly used to save typing
+ */
+var SOURCE_ALIASES = {
+  'osm': ['openstreetmap'],
   'oa': ['openaddresses'],
-  'openaddresses' : ['openaddresses'],
+  'gn': ['geonames'],
-  'qs'            : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'wof': ['whosonfirst']
-  'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
-  'osm'           : ['osmaddress', 'osmnode', 'osmway'],
-  'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
 };
 
-/**
+/*
- * This does not included alias layers, those are built separately
+ * Create an object that contains all sources or aliases. The key is the source or alias,
+ * the value is either that source, or the canonical name for that alias if it's an alias.
  */
-var LAYER_TO_TYPE = {
+var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES);
-  'venue': ['geoname','osmnode','osmway'],
+
-  'address': ['osmaddress','openaddresses'],
+/*
-  'country': ['admin0'],
+ * Layers
-  'region': ['admin1'],
+ */
-  'county': ['admin2'],
+
-  'locality': ['locality'],
+/*
-  'localadmin': ['local_admin'],
+ * A list of all layers in each source. This is used for convenience elswhere
-  'neighbourhood': ['neighborhood']
+ * and to determine when a combination of source and layer parameters is
+ * not going to match any records and will return no results.
+ */
+var LAYERS_BY_SOURCE = {
+ openstreetmap: [ 'address', 'venue' ],
+ openaddresses: [ 'address' ],
+ geonames: [ 'country', 'region', 'county', 'locality', 'venue' ],
+ whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region',
+   'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed']
 };
 
+/*
+ * A list of layer aliases that can be used to support specific use cases
+ * (like coarse geocoding) * or work around the fact that different sources
+ * may have layers that mean the same thing but have a different name
+ */
 var LAYER_ALIASES = {
-  'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin']
+  'coarse': LAYERS_BY_SOURCE.whosonfirst
 };
 
-var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE);
+// create a list of all layers by combining each entry from LAYERS_BY_SOURCE
+var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) {
+  return acc.concat(LAYERS_BY_SOURCE[key]);
+}, []));
 
 /*
- * derive the list of types, sources, and layers from above mappings
+ * Create the an object that has a key for each possible layer or alias,
+ * and returns either that layer, or all the layers in the alias
  */
-var TYPES   = Object.keys(TYPE_TO_SOURCE);
+var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES);
-var SOURCES = Object.keys(SOURCE_TO_TYPE);
-var LAYERS  = Object.keys(LAYER_TO_TYPE);
-
-var sourceAndLayerToType = function sourceAndLayerToType(source, layer) {
-  return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]);
-};
 
 module.exports = {
-  types: TYPES,
   sources: SOURCES,
   layers: LAYERS,
-  type_to_source: TYPE_TO_SOURCE,
+  source_mapping: SOURCE_MAPPING,
-  type_to_layer: TYPE_TO_LAYER,
+  layer_mapping: LAYER_MAPPING,
-  source_to_type: SOURCE_TO_TYPE,
+  layers_by_source: LAYERS_BY_SOURCE
-  layer_to_type: LAYER_TO_TYPE,
-  layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE,
-  source_and_layer_to_type: sourceAndLayerToType
 };

helper/types.js

@@ -1,43 +0,0 @@
-var type_mapping = require( '../helper/type_mapping' );
-var _ = require('lodash');
-
-/**
- * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched
- * This method decides how to combine all the preferences.
- *
- * @param {Array} clean_types
- * @returns {Array}
- */
-module.exports = function calculate_types(clean_types) {
-  //Check that at least one preference of types is defined
-  if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) {
-    throw new Error('clean_types should not be null or undefined');
-  }
-
-  /* the layers and source parameters are cumulative:
-   * perform a set intersection of their specified types
-   */
-  if (clean_types.from_layers || clean_types.from_sources) {
-    var types = type_mapping.types;
-
-    if (clean_types.from_layers) {
-      types = _.intersection(types, clean_types.from_layers);
-    }
-
-    if (clean_types.from_sources) {
-      types = _.intersection(types, clean_types.from_sources);
-    }
-
-    return types;
-  }
-
-  /*
-   * Type restrictions requested by the address parser should only be used
-   * if both the source and layers parameters are empty, so do this last
-   */
-  if (clean_types.from_text_parser) {
-    return clean_types.from_text_parser;
-  }
-
-  throw new Error('no types specified');
-};

middleware/_types.js

@@ -1,46 +0,0 @@
-var types_helper = require( '../helper/types' );
-
-/**
- * Validate the types specified to be searched.
- *
- * Elasticsearch interprets an empty array of types as "search anything" rather
- * than "search nothing", so in the case of an empty array, return an error
- * message instead of searching at all.
- */
-function middleware(req, res, next) {
-  req.clean = req.clean || {};
-
-  if (req.clean.hasOwnProperty('types')) {
-
-    try {
-      var types = types_helper(req.clean.types);
-
-      if ((types instanceof Array) && types.length === 0) {
-        var err = 'You have specified both the `sources` and `layers` ' +
-          'parameters in a combination that will return no results.';
-        req.errors.push( err );
-      }
-
-      else {
-        req.clean.type = types;
-      }
-
-    }
-
-    // @todo: refactor this flow, it is confusing as `types_helper()` can throw
-    // with an error "clean_types should not be null or undefined" which is
-    // not returned to the user yet the return value CAN trigger a user error.
-    // I would have liked to throw for BOTH cases and then handle the users errors
-    // inside the 'catch' but this is not possible.
-    // also: why are we deleting things from $clean?
-    catch (err) {
-      // this means there were no types specified
-      delete req.clean.types;
-    }
-
-  }
-
-  next();
-}
-
-module.exports = middleware;

middleware/confidenceScore.js

@@ -84,7 +84,7 @@ function computeConfidenceScore(req, mean, stdev, hit) {
 
 /*
  * Check for clearly mismatching properties in a result
- * zip code and state (admin1) are currently checked if present
+ * zip code and state (region) are currently checked if present
  *
  * @param {object|undefined} text
  * @param {object} hit
@@ -95,14 +95,15 @@ function checkForDealBreakers(req, hit) {
     return false;
   }
 
-  if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.admin1_abbr) {
+  if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.parent.region_a[0]) {
-    logger.debug('[confidence][deal-breaker]: state !== admin1_abbr');
+    logger.debug('[confidence][deal-breaker]: state !== region_a');
     return true;
   }
 
-  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address) &&
+  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) &&
-      req.clean.parsed_text.postalcode !== hit.address.zip) {
+      req.clean.parsed_text.postalcode !== hit.address_parts.zip) {
-    logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')');
+    logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode +
+      ' !== ' + hit.address_parts.zip + ')');
     return true;
   }
 }
@@ -155,8 +156,8 @@ function checkName(text, parsed_text, hit) {
  */
 function checkQueryType(text, hit) {
   if (check.assigned(text) && check.assigned(text.number) &&
-      (check.undefined(hit.address) ||
+      (check.undefined(hit.address_parts) ||
-      (check.assigned(hit.address) && check.undefined(hit.address.number)))) {
+      (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) {
     return 0;
   }
   return 1;
@@ -206,12 +207,12 @@ function propMatch(textProp, hitProp, expectEnriched) {
  * @param {string} [text.state]
  * @param {string} [text.country]
  * @param {object} hit
- * @param {object} [hit.address]
+ * @param {object} [hit.address_parts]
- * @param {string|number} [hit.address.number]
+ * @param {string|number} [hit.address_parts.number]
- * @param {string} [hit.address.street]
+ * @param {string} [hit.address_parts.street]
- * @param {string|number} [hit.address.zip]
+ * @param {string|number} [hit.address_parts.zip]
- * @param {string} [hit.admin1_abbr]
+ * @param {Array} [hit.parent.region_a]
- * @param {string} [hit.alpha3]
+ * @param {Array} [hit.parent.country_a]
  * @returns {number}
  */
 function checkAddress(text, hit) {
@@ -219,11 +220,11 @@ function checkAddress(text, hit) {
   var res = 0;
 
   if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) {
-    res += propMatch(text.number, (hit.address ? hit.address.number : null), false);
+    res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false);
-    res += propMatch(text.street, (hit.address ? hit.address.street : null), false);
+    res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false);
-    res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true);
+    res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true);
-    res += propMatch(text.state, hit.admin1_abbr, true);
+    res += propMatch(text.state, hit.parent.region_a[0], true);
-    res += propMatch(text.country, hit.alpha3, true);
+    res += propMatch(text.country, hit.parent.country_a[0], true);
 
     res /= checkCount;
   }

middleware/dedupe.js

@@ -39,22 +39,31 @@ function dedupeResults(req, res, next) {
  */
 function isDifferent(item1, item2) {
   try {
-    propMatch(item1, item2, 'admin1_abbr');
+    if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) {
-    propMatch(item1, item2, 'alpha3');
+      propMatch(item1.parent, item2.parent, 'region_a');
+      propMatch(item1.parent, item2.parent, 'country');
+      propMatch(item1.parent, item2.parent, 'locality');
+      propMatch(item1.parent, item2.parent, 'neighbourhood');
+    }
+    else if (item1.parent !== item2.parent) {
+      throw new Error('different');
+    }
 
     if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
-      propMatch(item1.name, item2.name, 'default');
+      for (var lang in item1.name) {
+	propMatch(item1.name, item2.name, lang);
+      }
     }
     else {
       propMatch(item1, item2, 'name');
     }
 
-    if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) {
+    if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) {
-      propMatch(item1.address, item2.address, 'number');
+      propMatch(item1.address_parts, item2.address_parts, 'number');
-      propMatch(item1.address, item2.address, 'street');
+      propMatch(item1.address_parts, item2.address_parts, 'street');
-      propMatch(item1.address, item2.address, 'zip');
+      propMatch(item1.address_parts, item2.address_parts, 'zip');
     }
-    else if (item1.address !== item2.address) {
+    else if (item1.address_parts !== item2.address_parts) {
       throw new Error('different');
     }
   }
@@ -77,7 +86,15 @@ function isDifferent(item1, item2) {
  * @throws {Error}
  */
 function propMatch(item1, item2, prop) {
-  if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) {
+  var prop1 = item1[prop];
+  var prop2 = item2[prop];
+
+  // in the case the property is an array (currently only in parent schema)
+  // simply take the 1st item. this will change in the near future to support multiple hierarchies
+  if (_.isArray(prop1)) { prop1 = prop1[0]; }
+  if (_.isArray(prop2)) { prop2 = prop2[0]; }
+
+  if (normalizeString(prop1) !== normalizeString(prop2)) {
     throw new Error('different');
   }
 }

middleware/localNamingConventions.js

@@ -1,5 +1,16 @@
+var check = require('check-types');
+var _ = require('lodash');
+
+var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL'];
 
 function setup() {
+  var api = require('pelias-config').generate().api;
+  var settings = api.localization;
+  if (settings && settings.flipNumberAndStreetCountries) {
+    var countries = settings.flipNumberAndStreetCountries;
+    flipNumberAndStreetCountries = _.uniq(flipNumberAndStreetCountries.concat(countries));
+  }
+
   return applyLocalNamingConventions;
 }
 
@@ -12,11 +23,15 @@ function applyLocalNamingConventions(req, res, next) {
 
   // loop through data items and flip relevant number/street
   res.data.filter(function(place){
-    // only relevant for German addresses
+    // relevant for some countries
-    if( 'DEU' !== place.alpha3 ){ return false; }
+    var flip = place.parent.country_a.some(function(country) {
-    if( !place.hasOwnProperty('address') ){ return false; }
+      return _.includes(flipNumberAndStreetCountries, country);
-    if( !place.address.hasOwnProperty('number') ){ return false; }
+    });
-    if( !place.address.hasOwnProperty('street') ){ return false; }
+    if (!flip){ return false; }
+    if( !place.hasOwnProperty('address_parts') ){ return false; }
+    if( !place.address_parts.hasOwnProperty('number') ){ return false; }
+    if( !place.address_parts.hasOwnProperty('street') ){ return false; }
+
     return true;
   })
   .forEach( flipNumberAndStreet );
@@ -24,11 +39,11 @@ function applyLocalNamingConventions(req, res, next) {
   next();
 }
 
-// DE address should have the housenumber and street name flipped
+// flip the housenumber and street name
 // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101'
 function flipNumberAndStreet(place) {
-  var standard = ( place.address.number + ' ' + place.address.street ),
+  var standard = ( place.address_parts.number + ' ' + place.address_parts.street ),
-      flipped  = ( place.address.street + ' ' + place.address.number );
+      flipped  = ( place.address_parts.street + ' ' + place.address_parts.number );
 
   // flip street name and housenumber
   if( place.name.default === standard ){

middleware/normalizeParentIds.js

@@ -0,0 +1,56 @@
+var logger = require('pelias-logger').get('api');
+var Document = require('pelias-model').Document;
+
+var placeTypes = require('../helper/placeTypes');
+
+/**
+ * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint.
+ * This should probably be moved to the import pipeline once we are happy with the way this works.
+ */
+
+function setup() {
+  return function (req, res, next) {
+    // do nothing if no result data set
+    if (!res || !res.data) {
+      return next();
+    }
+
+    res.data = res.data.map(normalizeParentIds);
+
+    next();
+  };
+}
+
+/**
+ * Update all parent ids in the admin hierarchy
+ *
+ * @param {object} place
+ * @return {object}
+ */
+function normalizeParentIds(place) {
+
+  if (place) {
+    placeTypes.forEach(function (placeType) {
+      if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) {
+        place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ];
+      }
+    });
+  }
+
+  return place;
+}
+
+/**
+ * Generate a valid Pelias ids from placetype and WOF id.
+ * Assumes all of the incoming ids are WOF ids.
+ *
+ * @param {string} placeType
+ * @param {number} id
+ * @return {string}
+ */
+function makeNewId(placeType, id) {
+  var doc = new Document('whosonfirst', placeType, id);
+  return doc.getGid();
+}
+
+module.exports = setup;

middleware/parseBBox.js

@@ -0,0 +1,38 @@
+var logger = require('pelias-logger').get('api');
+
+/**
+ * Parses the bounding box property in docs, if one is found
+ */
+
+function setup() {
+  return function (req, res, next) {
+    // do nothing if no result data set
+    if (!res || !res.data) {
+      return next();
+    }
+
+    res.data = res.data.map(parseBBox);
+
+    next();
+  };
+}
+
+/*
+ * Parse the bbox property and form an object
+ */
+function parseBBox(place) {
+
+  if (place && place.bounding_box) {
+    try {
+      place.bounding_box = JSON.parse(place.bounding_box);
+    }
+    catch (err) {
+      logger.error('Invalid bounding_box json string:', place);
+      delete place.bounding_box;
+    }
+  }
+
+  return place;
+}
+
+module.exports = setup;

middleware/renamePlacenames.js

@@ -1,66 +1,49 @@
-var extend = require('extend');
+var _ = require('lodash');
 
-/**
+var PARENT_PROPS = require('../helper/placeTypes');
- - P is a preferred English name
+
- - Q is a preferred name (in other languages)
+var ADDRESS_PROPS = {
- - V is a well-known (but unofficial) variant for the place
- (e.g. "New York City" for New York)
- - S is either a synonym or a colloquial name for the place
- (e.g. "Big Apple" for New York), or a version of the name which
- is stripped of accent characters.
- - A is an abbreviation or code for the place (e.g. "NYC" for New
- York)
- */
-// config mapping of old names to new ones
-var NAME_MAP = {
   'number': 'housenumber',
   'zip': 'postalcode',
-  'alpha3': 'country_a',
+  'street': 'street'
-  'admin0': 'country',
-  'admin1': 'region',
-  'admin1_abbr': 'region_a',
-  'admin2': 'county',
-  'local_admin': 'localadmin',
-  'neighborhood': 'neighbourhood'
 };
 
-function setup() {
 
+function setup() {
   return renamePlacenames;
 }
 
 function renamePlacenames(req, res, next) {
-
   // do nothing if no result data set
   if (!res || !res.data) {
     return next();
   }
 
-  // loop through data items and remap placenames
+  res.data = res.data.map(renameOneRecord);
-  res.data = res.data.map(renameProperties);
 
   next();
 }
 
-function renameProperties(place) {
+/*
-  var newPlace = {};
+ * Rename the fields in one record
-  Object.keys(place).forEach(function (property) {
+ */
-    if (property === 'address') {
+function renameOneRecord(place) {
-      extend(newPlace, renameProperties(place[property]));
+  if (place.address_parts) {
-    }
+    Object.keys(ADDRESS_PROPS).forEach(function (prop) {
-    else {
+      place[ADDRESS_PROPS[prop]] = place.address_parts[prop];
-      renameProperty(place, newPlace, property);
-    }
     });
-  return newPlace;
   }
 
-function renameProperty(oldObj, newObj, property) {
+  // merge the parent block into the top level object to flatten the structure
-  if (!oldObj.hasOwnProperty(property)) {
+  if (place.parent) {
-    return;
+    PARENT_PROPS.forEach(function (prop) {
+      place[prop] = place.parent[prop];
+      place[prop + '_a'] = place.parent[prop + '_a'];
+      place[prop + '_gid'] = place.parent[prop + '_id'];
+    });
   }
 
-  newObj[(NAME_MAP[property] || property)] = oldObj[property];
+  return place;
 }
 
 module.exports = setup;

package.json

@@ -13,7 +13,9 @@
     "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao",
     "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
     "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
-    "docs": "rm -r docs; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs"
+    "docs": "./bin/generate-docs",
+    "lint": "jshint .",
+    "validate": "npm ls"
   },
   "repository": {
     "type": "git",
@@ -34,37 +36,43 @@
   },
   "dependencies": {
     "addressit": "git://github.com/dianashk/addressit.git#temp",
-    "async": "^0.9.0",
+    "async": "^1.5.2",
-    "check-types": "^3.3.1",
+    "check-types": "^6.0.0",
     "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",
+    "elasticsearch": "^10.1.3",
     "express": "^4.8.8",
     "express-http-proxy": "^0.6.0",
-    "extend": "2.0.1",
+    "extend": "3.0.0",
-    "geojson": "^0.2.1",
+    "geojson": "^0.3.0",
     "geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "geopipes-elasticsearch-backend": "^0.2.0",
     "iso3166-1": "^0.2.3",
-    "lodash": "^3.10.1",
+    "lodash": "^4.5.0",
     "markdown": "0.5.0",
-    "morgan": "1.5.2",
+    "morgan": "1.7.0",
     "pelias-config": "^1.0.1",
-    "pelias-esclient": "0.0.25",
     "pelias-logger": "^0.0.8",
-    "pelias-query": "^5.0.0",
+    "pelias-model": "^3.1.0",
-    "pelias-schema": "1.0.0",
+    "pelias-query": "6.2.0",
-    "pelias-suggester-pipeline": "2.0.2",
+    "pelias-suggester-pipeline": "2.0.4",
-    "stats-lite": "^1.0.3",
+    "stats-lite": "1.0.3",
-    "through2": "0.6.5"
+    "through2": "2.0.1"
   },
   "devDependencies": {
     "ciao": "^0.6.0",
-    "istanbul": "^0.3.13",
+    "difflet": "^1.0.1",
+    "istanbul": "^0.4.2",
     "jshint": "^2.5.6",
-    "nsp": "^0.3.0",
+    "nsp": "^2.2.0",
-    "precommit-hook": "^1.0.7",
+    "precommit-hook": "^3.0.0",
     "proxyquire": "^1.4.0",
-    "tap-dot": "^1.0.0",
+    "tap-dot": "1.0.5",
-    "tape": "^2.13.4"
+    "tape": "^4.4.0"
-  }
+  },
+  "pre-commit": [
+    "lint",
+    "validate",
+    "test"
+  ]
 }

public/attribution.md

@@ -3,5 +3,5 @@
 * Data from
    * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/)
    * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation
-   * [Quattroshapes](https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md) under [CC-BY-2.0](https://creativecommons.org/licenses/by/2.0/)
    * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/)
+   * [WhosOnFirst](http://whosonfirst.mapzen.com) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md)

query/autocomplete.js

@@ -27,14 +27,14 @@ query.score( peliasQuery.view.address('street') );
 query.score( peliasQuery.view.address('postcode') );
 
 // admin components
-query.score( peliasQuery.view.admin('alpha3') );
+query.score( peliasQuery.view.admin('country') );
-query.score( peliasQuery.view.admin('admin0') );
+query.score( peliasQuery.view.admin('country_a') );
-query.score( peliasQuery.view.admin('admin1') );
+query.score( peliasQuery.view.admin('region') );
-query.score( peliasQuery.view.admin('admin1_abbr') );
+query.score( peliasQuery.view.admin('region_a') );
-query.score( peliasQuery.view.admin('admin2') );
+query.score( peliasQuery.view.admin('county') );
-query.score( peliasQuery.view.admin('local_admin') );
+query.score( peliasQuery.view.admin('localadmin') );
 query.score( peliasQuery.view.admin('locality') );
-query.score( peliasQuery.view.admin('neighborhood') );
+query.score( peliasQuery.view.admin('neighbourhood') );
 
 // scoring boost
 query.score( views.focus_selected_layers( views.ngrams_strict ) );

query/autocomplete_defaults.js

@@ -30,57 +30,57 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'phrase:slop': 2,
 
   'focus:function': 'linear',
-  'focus:offset': '10km',
+  'focus:offset': '0km',
   'focus:scale': '250km',
   'focus:decay': 0.5,
-  'focus:weight': 3,
+  'focus:weight': 10,
 
   'function_score:score_mode': 'avg',
   'function_score:boost_mode': 'multiply',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 2000,
 
-  'admin:alpha3:analyzer': 'standard',
+  'admin:country_a:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
+  'admin:country_a:field': 'parent.country_a',
-  'admin:alpha3:boost': 1000,
+  'admin:country_a:boost': 1000,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
+  'admin:country:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
+  'admin:country:field': 'parent.country',
-  'admin:admin0:boost': 800,
+  'admin:country:boost': 800,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
+  'admin:region:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
+  'admin:region:field': 'parent.region',
-  'admin:admin1:boost': 600,
+  'admin:region:boost': 600,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
+  'admin:region_a:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
+  'admin:region_a:field': 'parent.region_a',
-  'admin:admin1_abbr:boost': 600,
+  'admin:region_a:boost': 600,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
+  'admin:county:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
+  'admin:county:field': 'parent.county',
-  'admin:admin2:boost': 400,
+  'admin:county:boost': 400,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
+  'admin:localadmin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
+  'admin:localadmin:field': 'parent.localadmin',
-  'admin:local_admin:boost': 200,
+  'admin:localadmin:boost': 200,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 200,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
-  'admin:neighborhood:boost': 200,
+  'admin:neighbourhood:boost': 200,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',
@@ -90,6 +90,6 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'population:field': 'population',
   'population:modifier': 'log1p',
   'population:max_boost': 20,
-  'population:weight': 2
+  'population:weight': 3
 
 });

query/reverse.js

@@ -15,6 +15,7 @@ query.sort( peliasQuery.view.sort_distance );
 
 // non-scoring hard filters
 query.filter( peliasQuery.view.boundary_circle );
+query.filter( peliasQuery.view.sources );
 
 // --------------------------------
 
@@ -27,6 +28,9 @@ function generateQuery( clean ){
     vs.var( 'size', clean.querySize);
   }
 
+  // sources
+  vs.var( 'sources', clean.sources);
+
   // focus point to score by distance
   if( check.number(clean['point.lat']) &&
       check.number(clean['point.lon']) ){

query/reverse_defaults.js

@@ -39,48 +39,48 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 3,
 
-  'admin:alpha3:analyzer': 'standard',
+  'admin:country_a:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
+  'admin:country_a:field': 'parent.country_a',
-  'admin:alpha3:boost': 5,
+  'admin:country_a:boost': 5,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
+  'admin:country:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
+  'admin:country:field': 'parent.country',
-  'admin:admin0:boost': 4,
+  'admin:country:boost': 4,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
+  'admin:region:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
+  'admin:region:field': 'parent.region',
-  'admin:admin1:boost': 3,
+  'admin:region:boost': 3,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
+  'admin:region_a:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
+  'admin:region_a:field': 'parent.region_a',
-  'admin:admin1_abbr:boost': 3,
+  'admin:region_a:boost': 3,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
+  'admin:county:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
+  'admin:county:field': 'parent.county',
-  'admin:admin2:boost': 2,
+  'admin:county:boost': 2,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
+  'admin:localadmin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
+  'admin:localadmin:field': 'parent.localadmin',
-  'admin:local_admin:boost': 1,
+  'admin:localadmin:boost': 1,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 1,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
-  'admin:neighborhood:boost': 1,
+  'admin:neighbourhood:boost': 1,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',

query/search.js

@@ -25,19 +25,19 @@ query.score( peliasQuery.view.address('street') );
 query.score( peliasQuery.view.address('postcode') );
 
 // admin components
-query.score( peliasQuery.view.admin('alpha3') );
+query.score( peliasQuery.view.admin('country') );
-query.score( peliasQuery.view.admin('admin0') );
+query.score( peliasQuery.view.admin('country_a') );
-query.score( peliasQuery.view.admin('admin1') );
+query.score( peliasQuery.view.admin('region') );
-query.score( peliasQuery.view.admin('admin1_abbr') );
+query.score( peliasQuery.view.admin('region_a') );
-query.score( peliasQuery.view.admin('admin2') );
+query.score( peliasQuery.view.admin('county') );
-query.score( peliasQuery.view.admin('local_admin') );
+query.score( peliasQuery.view.admin('localadmin') );
 query.score( peliasQuery.view.admin('locality') );
-query.score( peliasQuery.view.admin('neighborhood') );
+query.score( peliasQuery.view.admin('neighbourhood') );
 
 // non-scoring hard filters
 query.filter( peliasQuery.view.boundary_circle );
 query.filter( peliasQuery.view.boundary_rect );
-
+query.filter( peliasQuery.view.sources );
 // --------------------------------
 
 /**
@@ -51,6 +51,9 @@ function generateQuery( clean ){
   // input text
   vs.var( 'input:name', clean.text );
 
+  // sources
+  vs.var( 'sources', clean.sources);
+
   // size
   if( clean.querySize ) {
     vs.var( 'size', clean.querySize );

query/search_defaults.js

@@ -39,48 +39,48 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 20,
 
-  'admin:alpha3:analyzer': 'standard',
+  'admin:country_a:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
+  'admin:country_a:field': 'parent.country_a',
-  'admin:alpha3:boost': 5,
+  'admin:country_a:boost': 5,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
+  'admin:country:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
+  'admin:country:field': 'parent.country',
-  'admin:admin0:boost': 4,
+  'admin:country:boost': 4,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
+  'admin:region:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
+  'admin:region:field': 'parent.region',
-  'admin:admin1:boost': 3,
+  'admin:region:boost': 3,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
+  'admin:region_a:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
+  'admin:region_a:field': 'parent.region_a',
-  'admin:admin1_abbr:boost': 3,
+  'admin:region_a:boost': 3,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
+  'admin:county:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
+  'admin:county:field': 'parent.county',
-  'admin:admin2:boost': 2,
+  'admin:county:boost': 2,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
+  'admin:localadmin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
+  'admin:localadmin:field': 'parent.localadmin',
-  'admin:local_admin:boost': 1,
+  'admin:localadmin:boost': 1,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 1,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
-  'admin:neighborhood:boost': 1,
+  'admin:neighbourhood:boost': 1,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',

query/text_parser.js

@@ -1,6 +1,20 @@
 
 var logger = require('pelias-logger').get('api');
-var adminFields = require('../helper/adminFields')();
+
+/*
+This list should only contain admin fields we are comfortable matching in the case
+when we can't identify parts of an address. This shouldn't contain fields like country_a
+or postalcode because we should only try to match those when we're sure that's what they are.
+ */
+var adminFields = [
+  'country',
+  'region',
+  'region_a',
+  'county',
+  'localadmin',
+  'locality',
+  'neighbourhood'
+];
 
 /**
   @todo: refactor me
@@ -48,17 +62,17 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
 
   // city
   if( parsed_text.hasOwnProperty('city') ){
-    vs.var( 'input:admin2', parsed_text.city );
+    vs.var( 'input:county', parsed_text.city );
   }
 
   // state
   if( parsed_text.hasOwnProperty('state') ){
-    vs.var( 'input:admin1_abbr', parsed_text.state );
+    vs.var( 'input:region_a', parsed_text.state );
   }
 
   // country
   if( parsed_text.hasOwnProperty('country') ){
-    vs.var( 'input:alpha3', parsed_text.country );
+    vs.var( 'input:country_a', parsed_text.country );
   }
 
   // ==== deal with the 'leftover' components ====
@@ -76,11 +90,10 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
   // if we have 'leftovers' then assign them to any fields which
   // currently don't have a value assigned.
   if( leftoversString.length ){
-    var unmatchedAdminFields = adminFields.slice();
 
     // cycle through fields and set fields which
     // are still currently unset
-    unmatchedAdminFields.forEach( function( key ){
+    adminFields.forEach( function( key ){
       if( !vs.isset( 'input:' + key ) ){
         vs.var( 'input:' + key, leftoversString );
       }

query/view/focus_selected_layers.js

@@ -22,10 +22,8 @@ module.exports = function( subview ){
     if( view && view.hasOwnProperty('function_score') ){
       view.function_score.filter = {
         'or': [
-          { 'type': { 'value': 'osmnode' } },
+          { 'term': { 'layer': 'venue' } },
-          { 'type': { 'value': 'osmway' } },
+          { 'term': { 'layer': 'address' } }
-          { 'type': { 'value': 'osmaddress' } },
-          { 'type': { 'value': 'openaddresses' } }
         ]
       };
     }

routes/v1.js

@@ -12,7 +12,6 @@ var sanitisers = {
 
 /** ----------------------- middleware ------------------------ **/
 var middleware = {
-  types: require('../middleware/_types'),
   calcSize: require('../middleware/sizeCalculator')
 };
 
@@ -35,7 +34,9 @@ var postProc = {
   localNamingConventions: require('../middleware/localNamingConventions'),
   renamePlacenames: require('../middleware/renamePlacenames'),
   geocodeJSON: require('../middleware/geocodeJSON'),
-  sendJSON: require('../middleware/sendJSON')
+  sendJSON: require('../middleware/sendJSON'),
+  parseBoundingBox: require('../middleware/parseBBox'),
+  normalizeParentIds: require('../middleware/normalizeParentIds')
 };
 
 /**
@@ -59,7 +60,6 @@ function addRoutes(app, peliasConfig) {
     ]),
     search: createRouter([
       sanitisers.search.middleware,
-      middleware.types,
       middleware.calcSize(),
       controllers.search(),
       postProc.distances('focus.point.'),
@@ -67,24 +67,26 @@ function addRoutes(app, peliasConfig) {
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
     autocomplete: createRouter([
       sanitisers.autocomplete.middleware,
-      middleware.types,
       controllers.search(null, require('../query/autocomplete')),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
     reverse: createRouter([
       sanitisers.reverse.middleware,
-      middleware.types,
       middleware.calcSize(),
       controllers.search(undefined, reverseQuery),
       postProc.distances('point.'),
@@ -94,6 +96,8 @@ function addRoutes(app, peliasConfig) {
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
@@ -102,6 +106,8 @@ function addRoutes(app, peliasConfig) {
       controllers.place(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),

sanitiser/_boundary_country.js

@@ -13,7 +13,7 @@ function sanitize(raw, clean) {
   if (check.assigned(country)){
 
     // must be valid string
-    if (!check.unemptyString(country)) {
+    if (!check.nonEmptyString(country)) {
       messages.errors.push('boundary.country is not a string');
     }
 

sanitiser/_categories.js

@@ -11,7 +11,7 @@ function sanitize( raw, clean ){
   clean.categories = [];
 
   // if categories string has been set
-  if( check.unemptyString( raw.categories ) ){
+  if( check.nonEmptyString( raw.categories ) ){
 
     // map input categories to valid format
     clean.categories = raw.categories.split(',')

sanitiser/_deprecate_quattroshapes.js

@@ -0,0 +1,41 @@
+var _ = require('lodash');
+
+/**
+  In the process of phasing out the 'quattroshapes' source in favour of 'whosonfirst'
+  we will emit a warning to users so they can begin upgrading their clients.
+
+  In the interim we will automatically rewrite all requests for quattroshapes to whosonfirst.
+
+  @todo: this is only temporary
+  @see: https://github.com/pelias/api/issues/442
+**/
+
+function sanitize( raw, clean, opts ) {
+  // error & warning messages
+  var messages = { errors: [], warnings: [] };
+
+  // only applicably when 'sources' param is privided
+  if( raw.hasOwnProperty('sources') ){
+
+    var sources = raw.sources.split(',');
+    if (_.includes(sources, 'quattroshapes') || _.includes(sources, 'qs')) {
+
+      // emit a warning message so users can transition.
+      messages.warnings.push('You are using Quattroshapes as a data source in this query. ' +
+       'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' +
+       'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' +
+       'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' +
+       'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' +
+       '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.');
+
+       // user requested 'quattroshapes', we will give them 'whosonfirst' instead.
+       sources = _.without(sources, 'quattroshapes', 'qs');
+       sources.push('whosonfirst');
+       raw.sources = sources.join(',');
+    }
+  }
+
+  return messages;
+}
+
+module.exports = sanitize;

sanitiser/_details.js

@@ -22,7 +22,7 @@ function sanitize( raw, clean ){
 // be lenient with 'truthy' values
 function isTruthy(val) {
   if( check.string( val ) ){
-    return _.contains( ['true', '1'], val );
+    return _.includes( ['true', '1'], val );
   }
 
   return val === 1 || val === true;

sanitiser/_flag_bool.js

@@ -44,7 +44,7 @@ function sanitize( raw, clean, opts ){
  * @returns {boolean}
  */
 function isTruthy(val) {
-  return _.contains( ['true', '1', 1, true], val );
+  return _.includes( ['true', '1', 1, true], val );
 }
 
 module.exports = setup;

sanitiser/_groups.js

@@ -10,7 +10,7 @@ var _ = require('lodash');
  * returns true if all are present, false if none are present, throws an exception otherwise
  */
 function optional_group(object, keys) {
-  var contained_in_object = _.contains.bind(null, Object.keys(object));
+  var contained_in_object = _.includes.bind(null, Object.keys(object));
 
   if (keys.every(contained_in_object)) {
     return true;
@@ -26,7 +26,7 @@ function optional_group(object, keys) {
  * An error will be thrown if any of the keys are missing from the object
  */
 function required_group(object, keys) {
-  var contained_in_object = _.contains.bind(null, Object.keys(object));
+  var contained_in_object = _.includes.bind(null, Object.keys(object));
 
   if (keys.every(contained_in_object)) {
     return true;

sanitiser/_ids.js

@@ -30,32 +30,25 @@ function sanitizeId(rawId, messages) {
   var id = parts.slice(2).join(ID_DELIM);
 
   // check if any parts of the gid are empty
-  if (_.contains([source, layer, id], '')) {
+  if (_.includes([source, layer, id], '')) {
     messages.errors.push( formatError(rawId) );
     return;
   }
 
-  if (!_.contains(type_mapping.sources, source)) {
+  if (!_.includes(type_mapping.sources, source)) {
     messages.errors.push( targetError(source, type_mapping.sources) );
     return;
   }
 
-  if (!_.contains(type_mapping.layers, layer)) {
+  if (!_.includes(type_mapping.layers, layer)) {
     messages.errors.push( targetError(layer, type_mapping.layers) );
     return;
   }
 
-  //TODO: remove this once we have a better set of layers for Geonames
-  var types;
-  if (source === 'gn' || source === 'geonames') {
-    types = ['geoname'];
-  } else {
-    types = type_mapping.source_and_layer_to_type(source, layer);
-  }
-
   return {
+    source: source,
+    layer: layer,
     id: id,
-    types: types
   };
 }
 
@@ -63,7 +56,7 @@ function sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
-  if (!check.unemptyString( raw.ids )) {
+  if (!check.nonEmptyString( raw.ids )) {
     messages.errors.push( lengthError);
     return messages;
   }
@@ -72,10 +65,10 @@ function sanitize( raw, clean ){
   var rawIds = raw.ids.split(',');
 
   // deduplicate
-  rawIds = _.unique(rawIds);
+  rawIds = _.uniq(rawIds);
 
   // ensure all elements are valid non-empty strings
-  if (!rawIds.every(check.unemptyString)) {
+  if (!rawIds.every(check.nonEmptyString)) {
       messages.errors.push( lengthError );
   }
 

sanitiser/_source.js

@@ -1,46 +0,0 @@
-
-var _ = require('lodash'),
-    check = require('check-types'),
-    sources_map = require( '../query/sources' );
-
-var ALL_SOURCES = Object.keys(sources_map),
-    ALL_SOURCES_JOINED = ALL_SOURCES.join(',');
-
-function sanitize( raw, clean ) {
-
-  // error & warning messages
-  var messages = { errors: [], warnings: [] };
-
-  // init clean.types (if not already init)
-  clean.types = clean.types || {};
-
-  // default case (no layers specified in GET params)
-  // don't even set the from_layers key in this case
-  if( check.unemptyString( raw.source ) ){
-
-    var sources = raw.source.split(',');
-
-    var invalid_sources = sources.filter(function(source) {
-      return !_.contains( ALL_SOURCES, source );
-    });
-
-    if( invalid_sources.length > 0 ){
-      invalid_sources.forEach( function( invalid ){
-        messages.errors.push('\'' + invalid + '\' is an invalid source parameter. Valid options: ' + ALL_SOURCES_JOINED);
-      });
-    }
-
-    else {
-      var types = sources.reduce(function(acc, source) {
-        return acc.concat(sources_map[source]);
-      }, []);
-
-      clean.types.from_source = types;
-    }
-
-  }
-
-  return messages;
-}
-
-module.exports = sanitize;

sanitiser/_sources_and_layers.js

@@ -0,0 +1,37 @@
+var _ = require( 'lodash' );
+var type_mapping = require( '../helper/type_mapping' );
+
+/*
+ * This sanitiser depends on clean.layers and clean.sources
+ * so it has to be run after those sanitisers have been run
+ */
+function sanitize( raw, clean ){
+  var messages = { errors: [], warnings: [] };
+
+  var possible_errors = [];
+  var at_least_one_valid_combination = false;
+
+  if (clean.layers && clean.sources) {
+    clean.sources.forEach(function(source) {
+      var layers_for_source = type_mapping.layers_by_source[source];
+      clean.layers.forEach(function(layer) {
+        if (_.includes(layers_for_source, layer)) {
+          at_least_one_valid_combination = true;
+        } else {
+          var message = 'You have specified both the `sources` and `layers` ' +
+            'parameters in a combination that will return no results: the ' +
+            source + ' source has nothing in the ' + layer + ' layer';
+          possible_errors.push(message);
+        }
+      });
+    });
+
+    if (!at_least_one_valid_combination) {
+      messages.errors = possible_errors;
+    }
+  }
+
+  return messages;
+}
+
+module.exports = sanitize;

sanitiser/_targets.js

@@ -1,34 +1,33 @@
-
 var _ = require('lodash'),
     check = require('check-types');
 
+function getValidKeys(mapping) {
+  return _.uniq(Object.keys(mapping)).join(',');
+}
+
 function setup( paramName, targetMap ) {
   return function( raw, clean ){
     return sanitize( raw, clean, {
       paramName: paramName,
       targetMap: targetMap,
-      targetMapKeysString: Object.keys(targetMap).join(',')
+      targetMapKeysString: getValidKeys(targetMap)
     });
   };
 }
 
 function sanitize( raw, clean, opts ) {
-
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
-  // init clean.types
-  clean.types = clean.types || {};
-
   // the string of targets (comma delimeted)
   var targetsString = raw[opts.paramName];
 
   // trim whitespace
-  if( check.unemptyString( targetsString ) ){
+  if( check.nonEmptyString( targetsString ) ){
     targetsString = targetsString.trim();
 
     // param must be a valid non-empty string
-    if( !check.unemptyString( targetsString ) ){
+    if( !check.nonEmptyString( targetsString ) ){
       messages.errors.push(
         opts.paramName + ' parameter cannot be an empty string. Valid options: ' + opts.targetMapKeysString
       );
@@ -51,19 +50,13 @@ function sanitize( raw, clean, opts ) {
 
       // only set types value when no error occured
       if( !messages.errors.length ){
-
+        clean[opts.paramName] = targets.reduce(function(acc, target) {
-        // store the values under a new key as 'clean.types.from_*'
-        var typesKey = 'from_' + opts.paramName;
-
-        // ?
-        clean.types[typesKey] = targets.reduce(function(acc, target) {
           return acc.concat(opts.targetMap[target]);
         }, []);
 
         // dedupe in case aliases expanded to common things or user typed in duplicates
-        clean.types[typesKey] = _.unique(clean.types[typesKey]);
+        clean[opts.paramName] = _.uniq(clean[opts.paramName]);
       }
-
     }
   }
 

sanitiser/_text.js

@@ -8,7 +8,7 @@ function sanitize( raw, clean ){
   var messages = { errors: [], warnings: [] };
 
   // invalid input 'text'
-  if( !check.unemptyString( raw.text ) ){
+  if( !check.nonEmptyString( raw.text ) ){
     messages.errors.push('invalid param \'text\': text length, must be >0');
   }
 
@@ -23,10 +23,6 @@ function sanitize( raw, clean ){
     if (check.assigned(parsed_text)) {
       clean.parsed_text = parsed_text;
     }
-
-    // try to set layers from query parser results
-    clean.types = clean.layers || {};
-    clean.types.from_text_parser = text_parser.get_layers(clean.text);
   }
 
   return messages;

sanitiser/_warn_quattroshapes.js

@@ -1,25 +0,0 @@
-var _ = require('lodash');
-
-function setup( paramName, targetMap ) {
-  return function( raw, clean ){
-    return sanitize( raw, clean );
-  };
-}
-
-function sanitize( raw, clean, opts ) {
-  // error & warning messages
-  var messages = { errors: [], warnings: [] };
-
-  if (_.includes(raw.sources, 'quattroshapes') || _.includes(raw.sources, 'qs')) {
-    messages.warnings.push( 'You are using Quattroshapes as a data source in this query. ' +
-     'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' +
-     'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' +
-     'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' +
-     'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' +
-     'be deprecated. If you have any questions, please email search@mapzen.com.');
-  }
-
-  return messages;
-}
-
-module.exports = setup;

sanitiser/reverse.js

@@ -2,10 +2,12 @@
 var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
+      quattroshapes_deprecation: require('../sanitiser/_deprecate_quattroshapes'),
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
-      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
-      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
-      quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(),
+      // depends on the layers and sources sanitisers, must be run after them
+      sources_and_layers: require('../sanitiser/_sources_and_layers'),
       size: require('../sanitiser/_size'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_reverse: require('../sanitiser/_geo_reverse'),

sanitiser/search.js

@@ -2,12 +2,14 @@ var type_mapping = require('../helper/type_mapping');
 
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
+      quattroshapes_deprecation: require('../sanitiser/_deprecate_quattroshapes'),
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
       size: require('../sanitiser/_size'),
-      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
-      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
-      quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(),
+      // depends on the layers and sources sanitisers, must be run after them
+      sources_and_layers: require('../sanitiser/_sources_and_layers'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),
       boundary_country: require('../sanitiser/_boundary_country'),

service/mget.js

@@ -24,7 +24,6 @@ function service( backend, query, cb ){
 
   // query new backend
   backend().client.mget( cmd, function( err, data ){
-
     // handle backend errors
     if( err ){ return cb( err ); }
 

src/backend.js

@@ -1,6 +1,6 @@
-
+var config = require( 'pelias-config' ).generate().esclient;
 var Backend = require('geopipes-elasticsearch-backend'),
-    client = require('pelias-esclient')(),
+    client = require('elasticsearch').Client(config),
     backends = {};
 
 function getBackend( index, type ){

test/ciao/place/basic_place.coffee

@@ -29,5 +29,5 @@ should.not.exist json.geocoding.errors
 should.not.exist json.geocoding.warnings
 
 #? inputs
-json.geocoding.query['ids'].should.eql [{ id: '1', types: [ 'geoname' ] }]
+json.geocoding.query['ids'].should.eql [{ id: '1', layer: 'venue', source: 'geonames' }]
 should.not.exist json.geocoding.query['size']

test/ciao/reverse/layers_alias_address.coffee

@@ -0,0 +1,33 @@
+
+#> layer alias
+path: '/v1/reverse?point.lat=1&point.lon=2&layers=address'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["address"]

test/ciao/reverse/layers_alias_coarse.coffee

@@ -30,5 +30,16 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.layers.should.eql [ "continent",
-json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+  "macrocountry",
+  "country",
+  "dependency",
+  "region",
+  "locality",
+  "localadmin",
+  "county",
+  "macrohood",
+  "neighbourhood",
+  "microhood",
+  "disputed"
+]

test/ciao/reverse/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_multiple.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
+json.geocoding.query.layers.should.eql ["country","region"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]

test/ciao/reverse/layers_single.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
+json.geocoding.query.layers.should.eql ["country"]
-json.geocoding.query['type'].should.eql ["admin0"]

test/ciao/reverse/sources_deprecation_warning.coffee

@@ -0,0 +1,34 @@
+
+#> quattroshapes is being phased out and so should emit a warning message
+path: '/v1/reverse?point.lat=1&point.lon=2&sources=qs'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.exist json.geocoding.warnings
+json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ]
+
+#? inputs
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes'

test/ciao/reverse/sources_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ]
+json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/sources_layers_invalid_combo.coffee

@@ -1,6 +1,6 @@
 
 #> sources and layers specified (invalid combo)
-path: '/v1/reverse?point.lat=1&point.lon=2&sources=quattroshapes&layers=address'
+path: '/v1/reverse?point.lat=1&point.lon=2&sources=whosonfirst&layers=address'
 
 #? 200 ok
 response.statusCode.should.be.equal 400
@@ -24,13 +24,12 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ]
+json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ]
 
 #? expected warnings
-json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ]
+should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
-json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.sources.should.eql ["whosonfirst"]
-should.not.exist json.geocoding.query['type']

test/ciao/reverse/sources_layers_valid_combo.coffee

@@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
-json.geocoding.query['type'].should.eql ["openaddresses"]
+json.geocoding.query.sources.should.eql ["openaddresses"]

test/ciao/reverse/sources_multiple.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"]
+json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"]
-json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"]

test/ciao/reverse/sources_single.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"]
+json.geocoding.query.sources.should.eql ["openstreetmap"]
-json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"]

test/ciao/search/layers_alias_address.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
-json.geocoding.query['type'].should.eql ["osmaddress","openaddresses"]

test/ciao/search/layers_alias_coarse.coffee

@@ -31,5 +31,16 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.layers.should.eql [ "continent",
-json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+  "macrocountry",
+  "country",
+  "dependency",
+  "region",
+  "locality",
+  "localadmin",
+  "county",
+  "macrohood",
+  "neighbourhood",
+  "microhood",
+  "disputed"
+]

test/ciao/search/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
-should.not.exist json.geocoding.query['type']

test/ciao/search/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
+should.not.exist json.geocoding.query['layers']
-should.not.exist json.geocoding.query['type']

test/ciao/search/layers_multiple.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
+json.geocoding.query.layers.should.eql ["country","region"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]

test/ciao/search/layers_single.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
+json.geocoding.query.layers.should.eql ["country"]
-json.geocoding.query['type'].should.eql ["admin0"]

test/ciao/search/sources_deprecation_warning.coffee

@@ -0,0 +1,35 @@
+
+#> quattroshapes is being phased out and so should emit a warning message
+path: '/v1/search?sources=qs&text=a'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.exist json.geocoding.warnings
+json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ]
+
+#? inputs
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes'

test/ciao/search/sources_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ]
+json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
-should.not.exist json.geocoding.query['type']

test/ciao/search/sources_layers_invalid_combo.coffee

@@ -1,6 +1,6 @@
 
 #> sources and layers specified (invalid combo)
-path: '/v1/search?text=a&sources=quattroshapes&layers=address'
+path: '/v1/search?text=a&sources=whosonfirst&layers=address'
 
 #? 200 ok
 response.statusCode.should.be.equal 400
@@ -24,14 +24,14 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ]
+json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ]
 
 #? expected warnings
-json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ]
+should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
-json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.sources.should.eql ["whosonfirst"]
 should.not.exist json.geocoding.query['type']

test/ciao/search/sources_layers_valid_combo.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
-json.geocoding.query['type'].should.eql ["openaddresses"]
+json.geocoding.query.sources.should.eql ["openaddresses"]

test/ciao/search/sources_multiple.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"]
+json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"]
-json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"]

test/ciao/search/sources_single.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"]
+json.geocoding.query.sources.should.eql ["openstreetmap"]
-json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"]

test/unit/controller/place.js

@@ -41,7 +41,7 @@ module.exports.tests.functional_success = function(test, common) {
 
   test('functional success', function(t) {
     var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
     var res = {
@@ -57,7 +57,7 @@ module.exports.tests.functional_success = function(test, common) {
         t.deepEqual(json.features, expected, 'values correctly mapped');
       }
     };
-    var req = { clean: { ids: [ {'id' : 123, types: [ 'a' ] } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, layers: [ 'a' ] } ] }, errors: [], warnings: [] };
     var next = function next() {
       t.equal(req.errors.length, 0, 'next was called without error');
       t.end();
@@ -70,10 +70,10 @@ module.exports.tests.functional_success = function(test, common) {
 module.exports.tests.functional_failure = function(test, common) {
   test('functional failure', function(t) {
     var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: ['b'] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
-    var req = { clean: { ids: [ {'id' : 123, types: [ 'b' ] } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] };
     var next = function( message ){
       t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler');
       t.end();

test/unit/controller/search.js

@@ -50,9 +50,11 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid1',
       _score: 10,
       _type: 'mytype1',
-      admin0: 'country1',
+      parent: {
-      admin1: 'state1',
+        country: ['country1'],
-      admin2: 'city1',
+        region: ['state1'],
+        county: ['city1']
+      },
       center_point: { lat: 100.1, lon: -50.5 },
       name: { default: 'test name1' },
       value: 1
@@ -61,9 +63,11 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid2',
       _score: 20,
       _type: 'mytype2',
-      admin0: 'country2',
+      parent: {
-      admin1: 'state2',
+        country: ['country2'],
-      admin2: 'city2',
+        region: ['state2'],
+        county: ['city2']
+      },
       center_point: { lat: 100.2, lon: -51.5 },
       name: { default: 'test name2' },
       value: 2

test/unit/fixture/autocomplete_linguistic_final_token.js

@@ -31,15 +31,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -60,17 +56,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -35,35 +35,25 @@ module.exports = {
                       'lat': 29.49136,
                       'lon': -82.50622
                     },
-                    'offset': '10km',
+                    'offset': '0km',
                     'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 3
+                'weight': 10
               }],
               'score_mode': 'avg',
               'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
-                    'type': {
+                    'term': {
-                      'value': 'osmnode'
+                      'layer': 'venue'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmway'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmaddress'
                     }
                   },
                   {
-                    'type': {
+                    'term': {
-                      'value': 'openaddresses'
+                      'layer': 'address'
                     }
                   }
                 ]
@@ -85,15 +75,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -114,17 +100,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -35,35 +35,25 @@ module.exports = {
                       'lat': 0,
                       'lon': 0
                     },
-                    'offset': '10km',
+                    'offset': '0km',
                     'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 3
+                'weight': 10
               }],
               'score_mode': 'avg',
               'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
-                    'type': {
+                    'term': {
-                      'value': 'osmnode'
+                      'layer': 'venue'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmway'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmaddress'
                     }
                   },
                   {
-                    'type': {
+                    'term': {
-                      'value': 'openaddresses'
+                      'layer': 'address'
                     }
                   }
                 ]
@@ -85,15 +75,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -114,17 +100,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_multiple_tokens.js

@@ -42,15 +42,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -71,17 +67,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_only.js

@@ -31,15 +31,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -60,17 +56,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_with_admin.js

@@ -20,7 +20,7 @@ module.exports = {
           'should': [
             {
               'match': {
-                'admin0': {
+                'parent.country': {
                   'analyzer': 'peliasAdmin',
                   'boost': 800,
                   'query': 'three'
@@ -29,7 +29,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin1': {
+                'parent.region': {
                   'analyzer': 'peliasAdmin',
                   'boost': 600,
                   'query': 'three'
@@ -38,7 +38,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin1_abbr': {
+                'parent.region_a': {
                   'analyzer': 'peliasAdmin',
                   'boost': 600,
                   'query': 'three'
@@ -47,7 +47,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin2': {
+                'parent.county': {
                   'analyzer': 'peliasAdmin',
                   'boost': 400,
                   'query': 'three'
@@ -56,7 +56,7 @@ module.exports = {
             },
             {
               'match': {
-                'local_admin': {
+                'parent.localadmin': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -65,7 +65,7 @@ module.exports = {
             },
             {
               'match': {
-                'locality': {
+                'parent.locality': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -74,7 +74,7 @@ module.exports = {
             },
             {
               'match': {
-                'neighborhood': {
+                'parent.neighbourhood': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -99,18 +99,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
               }
             },
             {
@@ -131,18 +127,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
-                    'weight': 2
+                    'weight': 3
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
               }
             }
           ]

test/unit/fixture/dedupe_elasticsearch_nonascii_results.js

@@ -7,9 +7,11 @@ module.exports = [
     'name': {
       'default': '万里长城万里长城'
     },
-    'country_a': 'CHN',
+    'parent': {
-    'country': 'China',
+      'country_a': ['CHN'],
-    'region': 'Beijing',
+      'country': ['China'],
+      'region': ['Beijing']
+    },
     'confidence': 0.733
   },
   {
@@ -20,9 +22,11 @@ module.exports = [
     'name': {
       'default': '万里长城'
     },
-    'country_a': 'CHN',
+    'parent': {
-    'country': 'China',
+      'country_a': ['CHN'],
-    'region': 'Beijing',
+      'country': ['China'],
+      'region': ['Beijing']
+    },
     'confidence': 0.733
   },
   {
@@ -33,12 +37,14 @@ module.exports = [
     'name': {
       'default': '万里花'
     },
-    'country_a': 'JPN',
+    'parent': {
-    'country': 'Japan',
+      'country_a': ['JPN'],
-    'region': 'Tokyo',
+      'country': ['Japan'],
-    'county': '豊島区',
+      'region': ['Tokyo'],
-    'locality': 'Tokyo',
+      'county': ['豊島区'],
-    'neighbourhood': '２丁目',
+      'locality': ['Tokyo'],
+      'neighbourhood': ['２丁目']
+    },
     'confidence': 0.646
   },
   {
@@ -49,13 +55,15 @@ module.exports = [
     'name': {
       'default': '万里加油站'
     },
-    'address': {
+    'address_parts': {
       'street': 'S308',
       'postalcode': '312044'
     },
-    'country_a': 'CHN',
+    'parent': {
-    'country': 'China',
+      'country_a': ['CHN'],
-    'region': 'Zhejiang',
+      'country': ['China'],
+      'region': ['Zhejiang']
+    },
     'confidence': 0.646
   }
 ];

test/unit/fixture/dedupe_elasticsearch_results.js

@@ -4,23 +4,79 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.039265
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'East Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
     'name': {
       'default': 'East Lampeter High School'
     },
-    'admin1': 'Pennsylvania',
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'venue',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  { // same as above, but change the neighbourhood
+    'center_point': {
+      'lon': -77.207456,
+      'lat': 41.039265
+    },
+    'address': {},
+    'parent': {
+      'localadmin': 'East Lampeter',
+      'region_a': 'PA',
+      'region': 'Pennsylvania',
       'locality': 'Smoketown',
-    'alpha3': 'USA',
+      'country_a': 'USA',
-    'admin2': 'Lancaster County',
+      'county': 'Lancaster County',
-    'admin0': 'United States',
+      'country': 'United States',
-    'neighborhood': 'Greenland',
+      'neighbourhood': 'Blueland' // ###
+    },
+    'name': {
+      'default': 'East Lampeter High School'
+    },
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'venue',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  { // same as #1, but change the locality
+    'center_point': {
+      'lon': -73.207456,
+      'lat': 42.039265
+    },
+    'address': {},
+    'parent': {
+      'localadmin': 'East Lampeter',
+      'region_a': 'PA',
+      'region': 'Pennsylvania',
+      'locality': 'Firetown', // ###
+      'country_a': 'USA',
+      'county': 'Lancaster County',
+      'country': 'United States',
+      'neighbourhood': 'Greenland'
+    },
+    'name': {
+      'default': 'East Lampeter High School'
+    },
     'category': [
       'education'
     ],
     '_id': '357321757',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -29,23 +85,25 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.039265
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'East Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
     'name': {
       'default': 'East Lampeter, High-School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
     '_id': '357321757',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -54,18 +112,20 @@ module.exports = [
       'lon': -76.23246,
       'lat': 39.99288
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'West Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'localadmin': ['West Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Lampeter'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Wheatland Mills']
+    },
     'name': {
       'default': 'Lampeter-Strasburg High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Lampeter',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Wheatland Mills',
     'category': [
       'education'
     ],
@@ -79,18 +139,20 @@ module.exports = [
       'lon': -76.20746,
       'lat': 40.03927
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'East Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
     'name': {
       'default': 'East Lampeter High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
@@ -104,23 +166,25 @@ module.exports = [
       'lon': -76.232457,
       'lat': 39.992877
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'West Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'region': ['Pennsylvania'],
+      'locality': ['Lampeter'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Wheatland Mills'],
+      'localadmin': ['West Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'Lampeter-Strasburg High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Lampeter',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Wheatland Mills',
     'category': [
       'education'
     ],
     '_id': '357294404',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -129,23 +193,25 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.038987
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'East Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland'],
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'East Lampeter School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
     '_id': '357283977',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.1036991,
     'confidence': 0.664
   },
@@ -154,18 +220,20 @@ module.exports = [
       'lon': -76.20746,
       'lat': 40.03899
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'East Lampeter',
+    'parent': {
-    'admin1_abbr': 'PA',
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland'],
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'East Lampeter School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
@@ -179,22 +247,24 @@ module.exports = [
       'lon': -94.167445,
       'lat': 38.762788
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'Polk',
+    'parent': {
-    'admin1_abbr': 'MO',
+      'region': ['Missouri'],
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Cass County'],
+      'country': ['United States'],
+      'localadmin': ['Polk'],
+      'region_a': ['MO']
+    },
     'name': {
       'default': 'Strasburg School'
     },
-    'admin1': 'Missouri',
-    'locality': 'Strasburg',
-    'alpha3': 'USA',
-    'admin2': 'Cass County',
-    'admin0': 'United States',
     'category': [
       'education'
     ],
     '_id': '358058986',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.0492544,
     'confidence': 0.658
   },
@@ -203,17 +273,19 @@ module.exports = [
       'lon': -78.36317,
       'lat': 38.98445
     },
-    'address': {},
+    'address_parts': {},
-    'admin1_abbr': 'VA',
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'Virginia',
+    'parent': {
-    'locality': 'Strasburg',
+      'region_a': ['VA'],
-    'alpha3': 'USA',
+      'region': ['Virginia'],
-    'admin2': 'Shenandoah County',
+      'locality': ['Strasburg'],
-    'admin0': 'United States',
+      'country_a': ['USA'],
-    'neighborhood': 'Strasburg Junction',
+      'county': ['Shenandoah County'],
+      'country': ['United States'],
+      'neighbourhood': ['Strasburg Junction']
+    },
     'category': [
       'education'
     ],
@@ -227,17 +299,19 @@ module.exports = [
       'lon': -100.16516,
       'lat': 46.13427
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'Strasburg',
-    'admin1_abbr': 'ND',
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'North Dakota',
+    'parent': {
-    'locality': 'Strasburg',
+      'localadmin': ['Strasburg'],
-    'alpha3': 'USA',
+      'region_a': ['ND'],
-    'admin2': 'Emmons County',
+      'region': ['North Dakota'],
-    'admin0': 'United States',
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Emmons County'],
+      'country': ['United States']
+    },
     'category': [
       'education'
     ],
@@ -251,22 +325,24 @@ module.exports = [
       'lon': -81.532392,
       'lat': 40.597578
     },
-    'address': {},
+    'address_parts': {},
-    'local_admin': 'Franklin',
-    'admin1_abbr': 'OH',
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'Ohio',
+    'parent': {
-    'locality': 'Strasburg',
+      'localadmin': ['Franklin'],
-    'alpha3': 'USA',
+      'region_a': ['OH'],
-    'admin2': 'Tuscarawas County',
+      'region': ['Ohio'],
-    'admin0': 'United States',
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Tuscarawas County'],
+      'country': ['United States']
+    },
     'category': [
       'education'
     ],
     '_id': '356646971',
-    '_type': 'osmway',
+    '_type': 'venue',
     '_score': 0.9724125,
     'confidence': 0.649
   }

test/unit/fixture/reverse_with_boundary_country.js

@@ -8,7 +8,7 @@ module.exports = {
           'must': [
             {
               'match': {
-                'alpha3': {
+                'parent.country_a': {
                   'analyzer': 'standard',
                   'query': 'ABC'
                 }

test/unit/fixture/reverse_with_source_filtering.js

@@ -0,0 +1,51 @@
+var vs = require('../../../query/reverse_defaults');
+
+module.exports = {
+  'query': {
+    'filtered': {
+      'query': {
+        'bool': {
+          'must': []
+        }
+      },
+      'filter': {
+        'bool': {
+          'must': [
+            {
+              'geo_distance': {
+                'distance': '500km',
+                'distance_type': 'plane',
+                'optimize_bbox': 'indexed',
+                '_cache': true,
+                'center_point': {
+                  'lat': 29.49136,
+                  'lon': -82.50622
+                }
+              }
+            },
+            {
+              'terms': {
+                'source': ['test']
+              }
+            }
+          ]
+        }
+      }
+    }
+  },
+  'sort': [
+    '_score',
+    {
+      '_geo_distance': {
+        'center_point': {
+          'lat': 29.49136,
+          'lon': -82.50622
+        },
+        'order': 'asc',
+        'distance_type': 'plane'
+      }
+    }
+  ],
+  'size': vs.size,
+  'track_scores': true
+};

test/unit/fixture/search_boundary_country.js

@@ -7,7 +7,7 @@ module.exports = {
           'must': [
             {
               'match': {
-                'alpha3': {
+                'parent.country_a': {
                   'analyzer': 'standard',
                   'query': 'ABC'
                 }
@@ -49,15 +49,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -78,15 +74,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_full_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,22 +66,18 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'address.number': {
+              'address_parts.number': {
                 'query': '123',
                 'boost': vs['address:housenumber:boost'],
                 'analyzer': vs['address:housenumber:analyzer']
@@ -93,7 +85,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.street': {
+              'address_parts.street': {
                 'query': 'main st',
                 'boost': vs['address:street:boost'],
                 'analyzer': vs['address:street:analyzer']
@@ -101,7 +93,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.zip': {
+              'address_parts.zip': {
                 'query': '10010',
                 'boost': vs['address:postcode:boost'],
                 'analyzer': vs['address:postcode:analyzer']
@@ -109,55 +101,55 @@ module.exports = {
             }
           }, {
             'match': {
-              'alpha3': {
+              'parent.country': {
-                'query': 'USA',
+                'query': 'new york',
-                'boost': vs['admin:alpha3:boost'],
+                'boost': vs['admin:country:boost'],
-                'analyzer': vs['admin:alpha3:analyzer']
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin0': {
+              'parent.country_a': {
-                'query': 'new york',
+                'query': 'USA',
-                'boost': vs['admin:admin0:boost'],
+                'boost': vs['admin:country_a:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+                'analyzer': vs['admin:country_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'new york',
-                'boost': vs['admin:admin1:boost'],
+                'boost': vs['admin:region:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'NY',
-                'boost': vs['admin:admin1_abbr:boost'],
+                'boost': vs['admin:region_a:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'new york',
-                'boost': vs['admin:admin2:boost'],
+                'boost': vs['admin:county:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'new york',
-                'boost': vs['admin:local_admin:boost'],
+                'boost': vs['admin:localadmin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'new york',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -165,10 +157,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'new york',
-                'boost': vs['admin:neighborhood:boost'],
+                'boost': vs['admin:neighbourhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_linguistic_bbox.js

@@ -39,15 +39,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -68,15 +64,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus_bbox.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus_null_island.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_only.js

@@ -39,15 +39,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -68,15 +64,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_viewport.js

@@ -77,18 +77,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
               }
             },
             {
@@ -109,18 +105,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
                     'weight': 2
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
               }
             }
           ]

test/unit/fixture/search_linguistic_viewport_min_diagonal.js

@@ -77,18 +77,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
               }
             },
             {
@@ -109,18 +105,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
                     'weight': 2
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
+                'boost_mode': 'replace'
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
               }
             }
           ]

test/unit/fixture/search_partial_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,62 +66,58 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'admin0': {
+              'parent.country': {
                 'query': 'new york',
-                'boost': vs['admin:admin0:boost'],
+                'boost': vs['admin:country:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'new york',
-                'boost': vs['admin:admin1:boost'],
+                'boost': vs['admin:region:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'new york',
-                'boost': vs['admin:admin1_abbr:boost'],
+                'boost': vs['admin:region_a:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'new york',
-                'boost': vs['admin:admin2:boost'],
+                'boost': vs['admin:county:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'new york',
-                'boost': vs['admin:local_admin:boost'],
+                'boost': vs['admin:localadmin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'new york',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -133,10 +125,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'new york',
-                'boost': vs['admin:neighborhood:boost'],
+                'boost': vs['admin:neighbourhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_regions_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,22 +66,18 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'address.number': {
+              'address_parts.number': {
                 'query': '1',
                 'boost': vs['address:housenumber:boost'],
                 'analyzer': vs['address:housenumber:analyzer']
@@ -93,7 +85,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.street': {
+              'address_parts.street': {
                 'query': 'water st',
                 'boost': vs['address:street:boost'],
                 'analyzer': vs['address:street:analyzer']
@@ -101,47 +93,47 @@ module.exports = {
             }
           }, {
             'match': {
-              'admin0': {
+              'parent.country': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin0:boost'],
+                'boost': vs['admin:country:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin1:boost'],
+                'boost': vs['admin:region:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'NY',
-                'boost': vs['admin:admin1_abbr:boost'],
+                'boost': vs['admin:region_a:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin2:boost'],
+                'boost': vs['admin:county:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'manhattan',
-                'boost': vs['admin:local_admin:boost'],
+                'boost': vs['admin:localadmin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'manhattan',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -149,10 +141,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'manhattan',
-                'boost': vs['admin:neighborhood:boost'],
+                'boost': vs['admin:neighbourhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/helper/adminFields.js

@@ -1,84 +0,0 @@
-var adminFields = require('../../../helper/adminFields');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = function(test, common) {
-  test('validate fields', function(t) {
-    t.assert(adminFields instanceof Function, 'adminFields is a function');
-    t.assert(adminFields() instanceof Array, 'adminFields() returns an array');
-    t.assert(adminFields().length > 0, 'adminFields array is not empty');
-    t.end();
-  });
-};
-
-module.exports.tests.lookupExistance = function(test, common) {
-  test('all expected fields in schema', function(t) {
-
-    var expectedFields = [
-      'one',
-      'two',
-      'three',
-      'four'
-    ];
-    var schema = { mappings: { _default_: { properties: {} } } };
-
-    // inject all expected fields into schema mock
-    expectedFields.forEach(function (field) {
-      schema.mappings._default_.properties[field] = {};
-    });
-
-    var res = adminFields(schema, expectedFields);
-
-    t.deepEquals(res, expectedFields, 'all expected fields are returned');
-    t.end();
-  });
-
-  test('some expected fields in schema', function(t) {
-
-    var expectedFields = [
-      'one',
-      'two',
-      'three',
-      'four'
-    ];
-    var schema = { mappings: { _default_: { properties: {} } } };
-
-    // inject only some of the expected fields into schema mock
-    expectedFields.slice(0, 3).forEach(function (field) {
-      schema.mappings._default_.properties[field] = {};
-    });
-
-    var res = adminFields(schema, expectedFields);
-
-    t.deepEquals(res, expectedFields.slice(0, 3), 'only matching expected fields are returned');
-    t.end();
-  });
-
-  test('no expected fields in schema', function(t) {
-
-    var schema = { mappings: { _default_: { properties: { foo: {} } } } };
-
-    var logErrorCalled = false;
-    var logger = {
-      error: function () {
-        logErrorCalled = true;
-      }};
-
-    var res = adminFields(schema, undefined, logger);
-
-    t.deepEquals(res, [], 'no admin fields found');
-    t.assert(logErrorCalled, 'log error called');
-    t.end();
-  });
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('adminFields: ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/helper/geojsonify.js

@@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) {
   var earth = [{
     '_type': 'geoname',
     '_id': '6295630',
+    'source': 'whosonfirst',
+    'layer': 'continent',
     'name': {
       'default': 'Earth'
     },
@@ -40,7 +42,9 @@ module.exports.tests.search = function(test, common) {
   var input = [
     {
       '_id': 'id1',
-      '_type': 'type1',
+      '_type': 'layer1',
+      'source': 'source1',
+      'layer': 'layer1',
       'center_point': {
         'lat': 51.5337144,
         'lon': -0.1069716
@@ -59,12 +63,6 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'test1',
       'locality': 'test2',
       'neighbourhood': 'test3',
-      'suggest': {
-        'input': [
-          '\'round midnight jazz and blues bar'
-        ],
-        'output': 'osmnode:2208150035'
-      },
       'category': [
         'food',
         'nightlife'
@@ -72,7 +70,9 @@ module.exports.tests.search = function(test, common) {
     },
     {
       '_id': 'id2',
-      '_type': 'type2',
+      '_type': 'layer2',
+      'source': 'source2',
+      'layer': 'layer2',
       'name': {
         'default': 'Blues Cafe'
       },
@@ -88,16 +88,12 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'test1',
       'locality': 'test2',
       'neighbourhood': 'test3',
-      'suggest': {
-        'input': [
-          'blues cafe'
-        ],
-        'output': 'osmway:147495160'
-      }
     },
     {
-      '_id': '34633854',
+      '_id': 'node:34633854',
-      '_type': 'osmway',
+      '_type': 'venue',
+      'source': 'openstreetmap',
+      'layer': 'venue',
       'name': {
         'default': 'Empire State Building'
       },
@@ -113,12 +109,6 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'Manhattan',
       'locality': 'New York',
       'neighbourhood': 'Koreatown',
-      'suggest': {
-        'input': [
-          'empire state building'
-        ],
-        'output': 'osmway:34633854'
-      },
       'category': [
         'tourism',
         'transport'
@@ -141,9 +131,9 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id1',
-          'gid': 'type1:type1:id1',
+          'gid': 'source1:layer1:id1',
-          'layer': 'type1',
+          'layer': 'layer1',
-          'source': 'type1',
+          'source': 'source1',
           'label': '\'Round Midnight Jazz and Blues Bar, test3, Angel',
           'name': '\'Round Midnight Jazz and Blues Bar',
           'country_a': 'GBR',
@@ -170,9 +160,9 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id2',
-          'gid': 'type2:type2:id2',
+          'gid': 'source2:layer2:id2',
-          'layer': 'type2',
+          'layer': 'layer2',
-          'source': 'type2',
+          'source': 'source2',
           'label': 'Blues Cafe, test3, Smithfield',
           'name': 'Blues Cafe',
           'country_a': 'GBR',
@@ -195,11 +185,11 @@ module.exports.tests.search = function(test, common) {
           ]
         },
         'properties': {
-          'id': '34633854',
+          'id': 'node:34633854',
-          'gid': 'osm:venue:34633854',
+          'gid': 'openstreetmap:venue:node:34633854',
           'layer': 'venue',
-          'source': 'osm',
+          'source': 'openstreetmap',
-          'label': 'Empire State Building, Manhattan, NY',
+          'label': 'Empire State Building, Manhattan, NY, USA',
           'name': 'Empire State Building',
           'country_a': 'USA',
           'country': 'United States',
@@ -219,6 +209,150 @@ module.exports.tests.search = function(test, common) {
     t.deepEqual(json, expected, 'all docs mapped');
     t.end();
   });
+
+  test('filtering out empty items', function (t) {
+    var input = [
+      {
+        'bounding_box': {
+          'min_lat': 40.6514712164,
+          'max_lat': 40.6737320588,
+          'min_lon': -73.8967895508,
+          'max_lon': -73.8665771484
+        },
+        'locality': [
+          'New York'
+        ],
+        'source': 'whosonfirst',
+        'layer': 'neighbourhood',
+        'population': 173198,
+        'popularity': 495,
+        'center_point': {
+          'lon': -73.881319,
+          'lat': 40.663303
+        },
+        'name': {
+          'default': 'East New York'
+        },
+        'source_id': '85816607',
+        'category': [],
+        '_id': '85816607',
+        '_type': 'neighbourhood',
+        '_score': 21.434,
+        'confidence': 0.888,
+        'country': [
+          'United States'
+        ],
+        'country_gid': [
+          '85633793'
+        ],
+        'country_a': [
+          'USA'
+        ],
+        'macroregion': [
+          'MacroRegion Name'
+        ],
+        'macroregion_gid': [
+          'MacroRegion Id'
+        ],
+        'macroregion_a': [
+          'MacroRegion Abbreviation'
+        ],
+        'region': [
+          'New York'
+        ],
+        'region_gid': [
+          '85688543'
+        ],
+        'region_a': [
+          'NY'
+        ],
+        'macrocounty': [
+          'MacroCounty Name'
+        ],
+        'macrocounty_gid': [
+          'MacroCounty Id'
+        ],
+        'macrocounty_a': [
+          'MacroCounty Abbreviation'
+        ],
+        'county': [
+          'Kings County'
+        ],
+        'county_gid': [
+          '102082361'
+        ],
+        'county_a': [
+          null
+        ],
+        'localadmin': [
+          'Brooklyn'
+        ],
+        'localadmin_gid': [
+          '404521211'
+        ],
+        'localadmin_a': [
+          null
+        ],
+        'locality_gid': [
+          '85977539'
+        ],
+        'locality_a': [
+          null
+        ],
+        'neighbourhood': [],
+        'neighbourhood_gid': []
+      }
+    ];
+
+    var expected = {
+      'type': 'FeatureCollection',
+      'bbox': [-73.8967895508, 40.6514712164, -73.8665771484, 40.6737320588],
+      'features': [
+        {
+          'type': 'Feature',
+          'properties': {
+            'id': '85816607',
+            'gid': 'whosonfirst:neighbourhood:85816607',
+            'layer': 'neighbourhood',
+            'source': 'whosonfirst',
+            'name': 'East New York',
+            'confidence': 0.888,
+            'country': 'United States',
+            'country_gid': '85633793',
+            'country_a': 'USA',
+            'macroregion': 'MacroRegion Name',
+            'macroregion_gid': 'MacroRegion Id',
+            'macroregion_a': 'MacroRegion Abbreviation',
+            'region': 'New York',
+            'region_gid': '85688543',
+            'region_a': 'NY',
+            'macrocounty': 'MacroCounty Name',
+            'macrocounty_gid': 'MacroCounty Id',
+            'macrocounty_a': 'MacroCounty Abbreviation',
+            'county': 'Kings County',
+            'county_gid': '102082361',
+            'localadmin': 'Brooklyn',
+            'localadmin_gid': '404521211',
+            'locality': 'New York',
+            'locality_gid': '85977539',
+            'label': 'East New York, Brooklyn, NY, USA'
+          },
+          'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],
+          'geometry': {
+            'type': 'Point',
+            'coordinates': [
+              -73.881319,
+              40.663303
+            ]
+          }
+        }
+      ]
+    };
+
+    var json = geojsonify.search( input );
+    t.deepEqual(json, expected, 'all wanted properties exposed');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/helper/labelGenerator_GBR.js

@@ -0,0 +1,87 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// GBR street address
+module.exports.tests.one_main_street_uk = function(test, common) {
+  test('one main street uk', function(t) {
+    var doc = {
+      'name': '1 Main St',
+      'housenumber': '1',
+      'street': 'Main St',
+      'postalcode': 'BT77 0BG',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Dungannon'
+    };
+    t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom');
+    t.end();
+  });
+};
+
+// GBR venue
+module.exports.tests.hackney_city_farm = function(test, common) {
+  test('hackney city farm', function(t) {
+    var doc = {
+      'name': 'Hackney City Farm',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Hackney',
+      'county': 'Greater London',
+      'locality': 'London',
+      'neighbourhood': 'Haggerston'
+    };
+    t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London');
+    t.end();
+  });
+};
+
+// GBR country
+module.exports.tests.wales = function(test, common) {
+  test('wales', function(t) {
+    var doc = {
+      'name': 'Wales',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Wales'
+    };
+    t.equal(generator(doc),'Wales, United Kingdom');
+    t.end();
+  });
+};
+
+// GBR macroregion
+module.exports.tests.macroregion_trumps_region = function(test, common) {
+  test('macroregion should trump region when none of neighbourhood, county, localadmin, locality are available', function(t) {
+    var doc = {
+      'name': 'Name',
+      'country_a': 'GBR',
+      'country': 'Country Name',
+      'macroregion': 'Macroregion Name',
+      'region': 'Region Name'
+    };
+
+    t.equal(generator(doc), 'Name, Macroregion Name, Country Name');
+    t.end();
+
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator (GBR): ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelGenerator_SGP.js

@@ -0,0 +1,51 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// SGP region
+module.exports.tests.north_west_singapore = function(test, common) {
+  test('north west singapore', function(t) {
+    var doc = {
+      'name': 'North West',
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'North West'
+    };
+    t.equal(generator(doc),'North West, Singapore');
+    t.end();
+  });
+};
+
+// SGP venue
+module.exports.tests.singapore_mcdonalds = function(test, common) {
+  test('singapore_mcdonalds', function(t) {
+    var doc = {
+      'name': 'McDonald\'s',
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'Central Singapore',
+      'locality': 'Singapore'
+    };
+    t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};