Merge pull request #471 from pelias/master

Merge master into staging

.travis.yml

@@ -1,6 +1,23 @@
+sudo: false
 language: node_js
-script: "npm run unit"
 node_js:
-  - "0.10"
-  - "0.12"
-sudo: false
+  - 0.10
+  - 0.12
+  - 4.4
+  - 5.8
+matrix:
+    allow_failures:
+        - node_js: 4.4
+        - node_js: 5.8
+env:
+  global:
+    - CXX=g++-4.8
+  matrix:
+    - TEST_SUITE=unit
+script: "npm run $TEST_SUITE"
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8

EXAMPLES.md

@@ -1,9 +0,0 @@
-
-#### Search:
-
-- Nearest Museums: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=museum&size=40
-- Nearest Hotels: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=hotel&size=40
-
-#### Autocomplete:
-
-- Local Neighborhoods: http://pelias.mapzen.com/suggest/nearby?lat=40.7259&lon=-73.9806&input=e&layers=neighborhood&size=40

bin/generate-docs

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+rm -r docs || true
+
+curl -s http://localhost:3100/v1 > /dev/null || die "Pelias server does not appear to be running \
+on http://localhost:3100, run npm start in another window before generating docs."
+
+cd test/ciao
+node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs

controller/place.js

@@ -1,4 +1,5 @@
 var service = { mget: require('../service/mget') };
+var logger = require('pelias-logger').get('api:controller:place');
 
 function setup( backend ){
 
@@ -13,40 +14,18 @@ function setup( backend ){
       return next();
     }
 
-    /* req.clean.ids contains an array of objects with id and types properties.
-     * types is an array of one or more types, since it can't always be known which single
-     * type a gid might belong to (osmnode and osmway both have source osm and layer venue).
-     *
-     * However, the mget Elasticsearch query only accepts a single type at a
-     * time.
-     *
-     * So, first create a new array that, has an entry
-     * with each type and id combination. This requires creating a new array with more entries
-     * than req.clean.ids in the case where entries have multiple types.
-     */
-
-    var recordsToReturn = req.clean.ids.reduce(function (acc, ids_element) {
-      ids_element.types.forEach(function(type) {
-        acc.push({
-          id: ids_element.id,
-          type: type
-        });
-      });
-      return acc;
-    }, []);
-
-    /*
-     * Next, map the list of records to an Elasticsearch mget query
-     */
-    var query = recordsToReturn.map( function(id) {
+    var query = req.clean.ids.map( function(id) {
       return {
         _index: 'pelias',
-        _type: id.type,
+        _type: id.layers,
         _id: id.id
       };
     });
 
+    logger.debug( '[ES req]', JSON.stringify(query) );
+
     service.mget( backend, query, function( err, docs ) {
+      console.log('err:' + err);
 
       // error handler
       if( err ){
@@ -56,6 +35,7 @@ function setup( backend ){
       else {
         res.data = docs;
       }
+      logger.debug('[ES response]', JSON.stringify(docs));
 
       next();
     });

controller/search.js

@@ -8,7 +8,6 @@ function setup( backend, query ){
   query = query || require('../query/search');
 
   function controller( req, res, next ){
-
     // do not run controller when a request
     // validation error has occurred.
     if( req.errors && req.errors.length ){
@@ -25,11 +24,13 @@ function setup( backend, query ){
       body: query( req.clean )
     };
 
-    // ?
-    if( req.clean.hasOwnProperty('type') ){
-      cmd.type = req.clean.type;
+    // use layers field for filtering by type
+    if( req.clean.hasOwnProperty('layers') ){
+      cmd.type = req.clean.layers;
     }
 
+    logger.debug( '[ES req]', JSON.stringify(cmd) );
+
     // query backend
     service.search( backend, cmd, function( err, docs, meta ){
 
@@ -42,7 +43,7 @@ function setup( backend, query ){
         res.data = docs;
         res.meta = meta;
       }
-
+      logger.debug('[ES response]', JSON.stringify(docs));
       next();
     });
 

helper/adminFields.js

@@ -1,45 +0,0 @@
-
-var _ = require('lodash'),
-    peliasSchema = require('pelias-schema'),
-    peliasLogger = require( 'pelias-logger' ).get( 'api' );
-
-var ADMIN_FIELDS = [
-  'admin0',
-  'admin1',
-  'admin1_abbr',
-  'admin2',
-  'local_admin',
-  'locality',
-  'neighborhood',
-  'address.zip'
-];
-
-/**
- * Get all admin fields that were expected and also found in schema
- *
- * @param {Object} [schema] optional: for testing only
- * @param {Array} [expectedFields] optional: for testing only
- * @param {Object} [logger] optional: for testing only
- * @returns {Array.<string>}
- */
-function getAvailableAdminFields(schema, expectedFields, logger) {
-
-  schema = schema || peliasSchema;
-  expectedFields = expectedFields || ADMIN_FIELDS;
-  logger = logger || peliasLogger;
-
-  var actualFields = Object.keys(schema.mappings._default_.properties);
-
-  // check if expected fields are actually in current schema
-  var available = expectedFields.filter(function (field) {
-    return _.contains( actualFields, field );
-  });
-
-  if (available.length === 0) {
-    logger.error('helper/adminFields: no expected admin fields found in schema');
-  }
-
-  return available;
-}
-
-module.exports = getAvailableAdminFields;

helper/geojsonify.js

@@ -11,48 +11,41 @@ var DETAILS_PROPS = [
   'housenumber',
   'street',
   'postalcode',
-  'country_a',
+  'confidence',
+  'distance',
   'country',
+  'country_id',
+  'country_a',
+  'macroregion',
+  'macroregion_id',
+  'macroregion_a',
   'region',
+  'region_id',
   'region_a',
+  'macrocounty',
+  'macrocounty_id',
+  'macrocounty_a',
   'county',
+  'county_id',
+  'county_a',
   'localadmin',
+  'localadmin_id',
+  'localadmin_a',
   'locality',
+  'locality_id',
+  'locality_a',
   'neighbourhood',
-  'confidence',
-  'distance'
+  'neighbourhood_id',
+  'bounding_box'
 ];
 
 
 function lookupSource(src) {
-  var sources = type_mapping.type_to_source;
-  return sources.hasOwnProperty(src._type) ? sources[src._type] : src._type;
+  return src.source;
 }
 
-/*
- * Use the type to layer mapping, except for Geonames, where having a full
- * Elasticsearch document source allows a more specific layer name to be chosen
- */
 function lookupLayer(src) {
-  if (src._type === 'geoname') {
-    if (_.contains(src.category, 'admin')) {
-      if (_.contains(src.category, 'admin:city')) { return 'locality'; }
-      if (_.contains(src.category, 'admin:admin1')) { return 'region'; }
-      if (_.contains(src.category, 'admin:admin2')) { return 'county'; }
-      return 'neighbourhood'; // this could also be 'local_admin'
-    }
-
-    if (src.name) { return 'venue'; }
-    if (src.address) { return 'address'; }
-  }
-
-  if (_.contains(type_mapping.types, src._type)) {
-    return type_mapping.type_to_layer[src._type];
-  }
-
-  logger.warn('[geojsonify]: could not map _type ', src._type);
-
-  return src._type;
+  return src.layer;
 }
 
 function geojsonifyPlaces( docs ){
@@ -64,11 +57,16 @@ function geojsonifyPlaces( docs ){
       return !!doc;
     });
 
+  // get all the bounding_box corners as well as single points
+  // to be used for computing the overall bounding_box for the FeatureCollection
+  var extentPoints = extractExtentPoints(geodata);
+
   // convert to geojson
-  var geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] });
+  var geojson             = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] });
+  var geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] });
 
   // bounding box calculations
-  computeBBox(geojson);
+  computeBBox(geojson, geojsonExtentPoints);
 
   return geojson;
 }
@@ -116,7 +114,40 @@ function addDetails(src, dst) {
  * @param {object} dst
  */
 function addLabel(src, dst) {
-  dst.label = labelGenerator(src);
+  dst.label = labelGenerator(dst);
+}
+
+
+/**
+ * Collect all points from the geodata.
+ * If an item is a single point, just use that.
+ * If an item has a bounding box, add two corners of the box as individual points.
+ *
+ * @param {Array} geodata
+ * @returns {Array}
+ */
+function extractExtentPoints(geodata) {
+  var extentPoints = [];
+  geodata.forEach(function (place) {
+    if (place.bounding_box) {
+      extentPoints.push({
+        lng: place.bounding_box.min_lon,
+        lat: place.bounding_box.min_lat
+      });
+      extentPoints.push({
+        lng: place.bounding_box.max_lon,
+        lat: place.bounding_box.max_lat
+      });
+    }
+    else {
+      extentPoints.push({
+        lng: place.lng,
+        lat: place.lat
+      });
+    }
+  });
+
+  return extentPoints;
 }
 
 /**
@@ -125,17 +156,17 @@ function addLabel(src, dst) {
  *
  * @param {object} geojson
  */
-function computeBBox(geojson) {
+function computeBBox(geojson, geojsonExtentPoints) {
   // @note: extent() sometimes throws Errors for unusual data
   // eg: https://github.com/pelias/pelias/issues/84
   try {
-    var bbox = extent( geojson );
+    var bbox = extent( geojsonExtentPoints );
     if( !!bbox ){
       geojson.bbox = bbox;
     }
   } catch( e ){
     console.error( 'bbox error', e.message, e.stack );
-    console.error( 'geojson', JSON.stringify( geojson, null, 2 ) );
+    console.error( 'geojson', JSON.stringify( geojsonExtentPoints, null, 2 ) );
   }
 }
 
@@ -149,8 +180,23 @@ function computeBBox(geojson) {
  */
 function copyProperties( source, props, dst ) {
   props.forEach( function ( prop ) {
+
     if ( source.hasOwnProperty( prop ) ) {
-      dst[prop] = source[prop];
+
+      // array value, take first item in array (at this time only used for admin values)
+      if (source[prop] instanceof Array) {
+        if (source[prop].length === 0) {
+          return;
+        }
+        if (source[prop][0]) {
+          dst[prop] = source[prop][0];
+        }
+      }
+
+      // simple value
+      else {
+        dst[prop] = source[prop];
+      }
     }
   });
 }

helper/labelGenerator.js

@@ -1,35 +1,74 @@
-
 var _ = require('lodash'),
-    check = require('check-types'),
-    schemas = require('./labelSchema.json');
+    schemas = require('./labelSchema');
 
 module.exports = function( record ){
+  var schema = getSchema(record.country_a);
+
+  var labelParts = getInitialLabel(record);
 
-  var labelParts = [ record.name.default ];
+  for (var key in schema) {
+    var valueFunction = schema[key];
 
-  var schema = schemas.default;
-  
-  if (record.country_a && record.country_a.length && schemas[record.country_a]) {
-    schema = schemas[record.country_a];
+    labelParts = valueFunction(record, labelParts);
   }
-  
-  var buildOutput = function(parts, schemaArr, record) {
-    for (var i=0; i<schemaArr.length; i++) {
-      var fieldValue = record[schemaArr[i]];
-      if (check.unemptyString(fieldValue) && !_.contains(parts, fieldValue)) {
-        parts.push( fieldValue );
-        return parts;
-      }
-    }
-    return parts;
-  };
 
-  for (var key in schema) {
-    labelParts = buildOutput(labelParts, schema[key], record);
+  // NOTE: while it may seem odd to call `uniq` on the list of label parts,
+  //  the effect is quite subtle.  Take, for instance, a result for "Lancaster, PA"
+  //  the pseudo-object is:
+  //  {
+  //    'name': 'Lancaster',
+  //    'locality': 'Lancaster',
+  //    'region_a': 'PA',
+  //    'country_a': 'USA'
+  //  }
+  //
+  //  the code up to this point generates the label:
+  //  `Lancaster, Lancaster, PA, USA`
+  //
+  //  then the `unique` call reduces this to:
+  //  `Lancaster, PA, USA`
+  //
+  //  this code doesn't have the same effect in the case of a venue or address
+  //  where the `name` field would contain the address or name of a point-of-interest
+  //
+  //  Also see https://github.com/pelias/api/issues/429 for other ways that this is bad
+  //
+  // de-dupe, join, trim
+  return _.uniq( labelParts ).join(', ').trim();
+
+};
+
+function getSchema(country_a) {
+  if (country_a && country_a.length && schemas[country_a]) {
+    return schemas[country_a];
+  }
+
+  return schemas.default;
+
+}
+
+// helper function that sets a default label for non-US/CA regions
+// this is a very special case
+function getInitialLabel(record) {
+  if (isRegion(record.layer) &&
+      isGeonamesOrWhosOnFirst(record.source) &&
+      isUSAOrCAN(record.country_a)) {
+    return [];
   }
 
-  // de-dupe outputs
-  labelParts = _.unique( labelParts );
+  return [record.name];
+
+}
+
+function isRegion(layer) {
+  return 'region' === layer;
+}
+
+function isUSAOrCAN(country_a) {
+  return 'USA' === country_a || 'CAN' === country_a;
+}
+
+function isGeonamesOrWhosOnFirst(source) {
+  return 'geonames' === source || 'whosonfirst' === source;
 
-  return labelParts.join(', ').trim();
-};
+}

helper/labelSchema.js

@@ -0,0 +1,66 @@
+var _ = require('lodash'),
+    check = require('check-types');
+
+module.exports = {
+  'USA': {
+    'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county']),
+    'regional': getUsState,
+    'country': getFirstProperty(['country_a'])
+  },
+  'GBR': {
+    'local': getFirstProperty(['neighbourhood', 'county', 'localadmin', 'locality', 'macroregion', 'region']),
+    'regional': getFirstProperty(['county','country','region'])
+  },
+  'SGP': {
+    'local': getFirstProperty(['neighbourhood', 'region', 'county', 'localadmin', 'locality']),
+    'regional': getFirstProperty(['county','country','region'])
+  },
+  'SWE': {
+    'local': getFirstProperty(['neighbourhood', 'region', 'county', 'localadmin', 'locality']),
+    'regional': getFirstProperty(['country'])
+  },
+  'default': {
+    'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county', 'macroregion', 'region']),
+    'regional': getFirstProperty(['country'])
+  }
+};
+
+// find the first field of record that has a non-empty value that's not already in labelParts
+function getFirstProperty(fields) {
+  return function(record, labelParts) {
+    for (var i = 0; i < fields.length; i++) {
+      var fieldValue = record[fields[i]];
+
+      if (check.nonEmptyString(fieldValue) && !_.includes(labelParts, fieldValue)) {
+        labelParts.push( fieldValue );
+        return labelParts;
+      }
+
+    }
+
+    return labelParts;
+
+  };
+
+}
+
+// this function is exclusively used for figuring out which field to use for US States
+// 1.  if a US state is the most granular bit of info entered, the label should contain
+//  the full state name, eg: Pennsylvania, USA
+// 2.  otherwise, the state abbreviation should be used, eg: Lancaster, PA, USA
+// 3.  if for some reason the abbreviation isn't available, use the full state name
+function getUsState(record, labelParts) {
+  if ('region' === record.layer && record.region) {
+    // add full state name when state is the most granular piece of info
+    labelParts.push(record.region);
+  } else if (record.region_a) {
+    // otherwise just add the region code when available
+    labelParts.push(record.region_a);
+  } else if (record.region) {
+    // add the full name when there's no region code available ()
+    labelParts.push(record.region);
+  }
+
+  return labelParts;
+
+}

helper/labelSchema.json

@@ -1,22 +0,0 @@
-{
-  "USA": {
-    "local": ["localadmin", "locality", "neighbourhood", "county"],
-    "regional": ["region_a", "region", "country"]
-  },
-  "GBR": {
-    "local": ["neighbourhood", "county", "localadmin", "locality", "region"],
-    "regional": ["county","country","region"]
-  },
-  "SGP": {
-    "local": ["neighbourhood", "region", "county", "localadmin", "locality"],
-    "regional": ["county","country","region"]
-  },
-  "SWE": {
-    "local": ["neighbourhood", "region", "county", "localadmin", "locality"],
-    "regional": ["country"]
-  },
-  "default": {
-    "local": ["localadmin", "locality", "neighbourhood", "county", "region"],
-    "regional": ["country"]
-  }
-}

helper/text_parser.js

@@ -15,7 +15,7 @@ module.exports = {};
 module.exports.get_layers = function get_layers(query) {
   if (query.length <= 3 ) {
     // no address parsing required
-    return type_mapping.layer_with_aliases_to_type.coarse;
+    return type_mapping.layer_mapping.coarse;
   }
 };
 

helper/type_mapping.js

@@ -1,87 +1,81 @@
 var extend = require('extend'),
   _ = require('lodash');
 
-var TYPE_TO_SOURCE = {
-  'geoname': 'gn',
-  'osmnode': 'osm',
-  'osmway': 'osm',
-  'admin0': 'qs',
-  'admin1': 'qs',
-  'admin2': 'qs',
-  'neighborhood': 'qs',
-  'locality': 'qs',
-  'local_admin': 'qs',
-  'osmaddress': 'osm',
-  'openaddresses': 'oa'
-};
+function addStandardTargetsToAliases(standard, aliases) {
+  var combined = _.extend({}, aliases);
+  standard.forEach(function(target) {
+    if (combined[target] === undefined) {
+      combined[target] = [target];
+    }
+  });
+
+  return combined;
+}
 
 /*
- * This doesn't include alias layers such as coarse
+ * Sources
  */
-var TYPE_TO_LAYER = {
-  'geoname': 'venue',
-  'osmnode': 'venue',
-  'osmway': 'venue',
-  'admin0': 'country',
-  'admin1': 'region',
-  'admin2': 'county',
-  'neighborhood': 'neighbourhood',
-  'locality': 'locality',
-  'local_admin': 'localadmin',
-  'osmaddress': 'address',
-  'openaddresses': 'address'
-};
 
-var SOURCE_TO_TYPE = {
-  'gn'            : ['geoname'],
-  'geonames'      : ['geoname'],
-  'oa'            : ['openaddresses'],
-  'openaddresses' : ['openaddresses'],
-  'qs'            : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
-  'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
-  'osm'           : ['osmaddress', 'osmnode', 'osmway'],
-  'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
+// a list of all sources
+var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'whosonfirst'];
+
+/*
+ * A list of alternate names for sources, mostly used to save typing
+ */
+var SOURCE_ALIASES = {
+  'osm': ['openstreetmap'],
+  'oa': ['openaddresses'],
+  'gn': ['geonames'],
+  'wof': ['whosonfirst']
 };
 
-/**
- * This does not included alias layers, those are built separately
+/*
+ * Create an object that contains all sources or aliases. The key is the source or alias,
+ * the value is either that source, or the canonical name for that alias if it's an alias.
  */
-var LAYER_TO_TYPE = {
-  'venue': ['geoname','osmnode','osmway'],
-  'address': ['osmaddress','openaddresses'],
-  'country': ['admin0'],
-  'region': ['admin1'],
-  'county': ['admin2'],
-  'locality': ['locality'],
-  'localadmin': ['local_admin'],
-  'neighbourhood': ['neighborhood']
+var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES);
+
+/*
+ * Layers
+ */
+
+/*
+ * A list of all layers in each source. This is used for convenience elswhere
+ * and to determine when a combination of source and layer parameters is
+ * not going to match any records and will return no results.
+ */
+var LAYERS_BY_SOURCE = {
+ openstreetmap: [ 'address', 'venue' ],
+ openaddresses: [ 'address' ],
+ geonames: [ 'country', 'region', 'county', 'locality', 'venue' ],
+ whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region',
+   'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed']
 };
 
+/*
+ * A list of layer aliases that can be used to support specific use cases
+ * (like coarse geocoding) * or work around the fact that different sources
+ * may have layers that mean the same thing but have a different name
+ */
 var LAYER_ALIASES = {
-  'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin']
+  'coarse': LAYERS_BY_SOURCE.whosonfirst
 };
 
-var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE);
+// create a list of all layers by combining each entry from LAYERS_BY_SOURCE
+var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) {
+  return acc.concat(LAYERS_BY_SOURCE[key]);
+}, []));
 
 /*
- * derive the list of types, sources, and layers from above mappings
+ * Create the an object that has a key for each possible layer or alias,
+ * and returns either that layer, or all the layers in the alias
  */
-var TYPES   = Object.keys(TYPE_TO_SOURCE);
-var SOURCES = Object.keys(SOURCE_TO_TYPE);
-var LAYERS  = Object.keys(LAYER_TO_TYPE);
-
-var sourceAndLayerToType = function sourceAndLayerToType(source, layer) {
-  return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]);
-};
+var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES);
 
 module.exports = {
-  types: TYPES,
   sources: SOURCES,
   layers: LAYERS,
-  type_to_source: TYPE_TO_SOURCE,
-  type_to_layer: TYPE_TO_LAYER,
-  source_to_type: SOURCE_TO_TYPE,
-  layer_to_type: LAYER_TO_TYPE,
-  layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE,
-  source_and_layer_to_type: sourceAndLayerToType
+  source_mapping: SOURCE_MAPPING,
+  layer_mapping: LAYER_MAPPING,
+  layers_by_source: LAYERS_BY_SOURCE
 };

helper/types.js

@@ -1,43 +0,0 @@
-var type_mapping = require( '../helper/type_mapping' );
-var _ = require('lodash');
-
-/**
- * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched
- * This method decides how to combine all the preferences.
- *
- * @param {Array} clean_types
- * @returns {Array}
- */
-module.exports = function calculate_types(clean_types) {
-  //Check that at least one preference of types is defined
-  if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) {
-    throw new Error('clean_types should not be null or undefined');
-  }
-
-  /* the layers and source parameters are cumulative:
-   * perform a set intersection of their specified types
-   */
-  if (clean_types.from_layers || clean_types.from_sources) {
-    var types = type_mapping.types;
-
-    if (clean_types.from_layers) {
-      types = _.intersection(types, clean_types.from_layers);
-    }
-
-    if (clean_types.from_sources) {
-      types = _.intersection(types, clean_types.from_sources);
-    }
-
-    return types;
-  }
-
-  /*
-   * Type restrictions requested by the address parser should only be used
-   * if both the source and layers parameters are empty, so do this last
-   */
-  if (clean_types.from_text_parser) {
-    return clean_types.from_text_parser;
-  }
-
-  throw new Error('no types specified');
-};

middleware/_types.js

@@ -1,46 +0,0 @@
-var types_helper = require( '../helper/types' );
-
-/**
- * Validate the types specified to be searched.
- *
- * Elasticsearch interprets an empty array of types as "search anything" rather
- * than "search nothing", so in the case of an empty array, return an error
- * message instead of searching at all.
- */
-function middleware(req, res, next) {
-  req.clean = req.clean || {};
-
-  if (req.clean.hasOwnProperty('types')) {
-
-    try {
-      var types = types_helper(req.clean.types);
-
-      if ((types instanceof Array) && types.length === 0) {
-        var err = 'You have specified both the `sources` and `layers` ' +
-          'parameters in a combination that will return no results.';
-        req.errors.push( err );
-      }
-
-      else {
-        req.clean.type = types;
-      }
-
-    }
-
-    // @todo: refactor this flow, it is confusing as `types_helper()` can throw
-    // with an error "clean_types should not be null or undefined" which is
-    // not returned to the user yet the return value CAN trigger a user error.
-    // I would have liked to throw for BOTH cases and then handle the users errors
-    // inside the 'catch' but this is not possible.
-    // also: why are we deleting things from $clean?
-    catch (err) {
-      // this means there were no types specified
-      delete req.clean.types;
-    }
-
-  }
-
-  next();
-}
-
-module.exports = middleware;

middleware/confidenceScore.js

@@ -84,7 +84,7 @@ function computeConfidenceScore(req, mean, stdev, hit) {
 
 /*
  * Check for clearly mismatching properties in a result
- * zip code and state (admin1) are currently checked if present
+ * zip code and state (region) are currently checked if present
  *
  * @param {object|undefined} text
  * @param {object} hit
@@ -95,14 +95,15 @@ function checkForDealBreakers(req, hit) {
     return false;
   }
 
-  if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.admin1_abbr) {
-    logger.debug('[confidence][deal-breaker]: state !== admin1_abbr');
+  if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.parent.region_a[0]) {
+    logger.debug('[confidence][deal-breaker]: state !== region_a');
     return true;
   }
 
-  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address) &&
-      req.clean.parsed_text.postalcode !== hit.address.zip) {
-    logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')');
+  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) &&
+      req.clean.parsed_text.postalcode !== hit.address_parts.zip) {
+    logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode +
+      ' !== ' + hit.address_parts.zip + ')');
     return true;
   }
 }
@@ -155,8 +156,8 @@ function checkName(text, parsed_text, hit) {
  */
 function checkQueryType(text, hit) {
   if (check.assigned(text) && check.assigned(text.number) &&
-      (check.undefined(hit.address) ||
-      (check.assigned(hit.address) && check.undefined(hit.address.number)))) {
+      (check.undefined(hit.address_parts) ||
+      (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) {
     return 0;
   }
   return 1;
@@ -206,12 +207,12 @@ function propMatch(textProp, hitProp, expectEnriched) {
  * @param {string} [text.state]
  * @param {string} [text.country]
  * @param {object} hit
- * @param {object} [hit.address]
- * @param {string|number} [hit.address.number]
- * @param {string} [hit.address.street]
- * @param {string|number} [hit.address.zip]
- * @param {string} [hit.admin1_abbr]
- * @param {string} [hit.alpha3]
+ * @param {object} [hit.address_parts]
+ * @param {string|number} [hit.address_parts.number]
+ * @param {string} [hit.address_parts.street]
+ * @param {string|number} [hit.address_parts.zip]
+ * @param {Array} [hit.parent.region_a]
+ * @param {Array} [hit.parent.country_a]
  * @returns {number}
  */
 function checkAddress(text, hit) {
@@ -219,11 +220,11 @@ function checkAddress(text, hit) {
   var res = 0;
 
   if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) {
-    res += propMatch(text.number, (hit.address ? hit.address.number : null), false);
-    res += propMatch(text.street, (hit.address ? hit.address.street : null), false);
-    res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true);
-    res += propMatch(text.state, hit.admin1_abbr, true);
-    res += propMatch(text.country, hit.alpha3, true);
+    res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false);
+    res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false);
+    res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true);
+    res += propMatch(text.state, hit.parent.region_a[0], true);
+    res += propMatch(text.country, hit.parent.country_a[0], true);
 
     res /= checkCount;
   }

middleware/dedupe.js

@@ -39,22 +39,31 @@ function dedupeResults(req, res, next) {
  */
 function isDifferent(item1, item2) {
   try {
-    propMatch(item1, item2, 'admin1_abbr');
-    propMatch(item1, item2, 'alpha3');
+    if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) {
+      propMatch(item1.parent, item2.parent, 'region_a');
+      propMatch(item1.parent, item2.parent, 'country');
+      propMatch(item1.parent, item2.parent, 'locality');
+      propMatch(item1.parent, item2.parent, 'neighbourhood');
+    }
+    else if (item1.parent !== item2.parent) {
+      throw new Error('different');
+    }
 
     if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
-      propMatch(item1.name, item2.name, 'default');
+      for (var lang in item1.name) {
+	propMatch(item1.name, item2.name, lang);
+      }
     }
     else {
       propMatch(item1, item2, 'name');
     }
 
-    if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) {
-      propMatch(item1.address, item2.address, 'number');
-      propMatch(item1.address, item2.address, 'street');
-      propMatch(item1.address, item2.address, 'zip');
+    if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) {
+      propMatch(item1.address_parts, item2.address_parts, 'number');
+      propMatch(item1.address_parts, item2.address_parts, 'street');
+      propMatch(item1.address_parts, item2.address_parts, 'zip');
     }
-    else if (item1.address !== item2.address) {
+    else if (item1.address_parts !== item2.address_parts) {
       throw new Error('different');
     }
   }
@@ -77,7 +86,15 @@ function isDifferent(item1, item2) {
  * @throws {Error}
  */
 function propMatch(item1, item2, prop) {
-  if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) {
+  var prop1 = item1[prop];
+  var prop2 = item2[prop];
+
+  // in the case the property is an array (currently only in parent schema)
+  // simply take the 1st item. this will change in the near future to support multiple hierarchies
+  if (_.isArray(prop1)) { prop1 = prop1[0]; }
+  if (_.isArray(prop2)) { prop2 = prop2[0]; }
+
+  if (normalizeString(prop1) !== normalizeString(prop2)) {
     throw new Error('different');
   }
 }
@@ -96,4 +113,4 @@ function normalizeString(str) {
 }
 
 
-module.exports = setup;
+module.exports = setup;

middleware/localNamingConventions.js

@@ -1,5 +1,16 @@
+var check = require('check-types');
+var _ = require('lodash');
+
+var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL'];
 
 function setup() {
+  var api = require('pelias-config').generate().api;
+  var settings = api.localization;
+  if (settings && settings.flipNumberAndStreetCountries) {
+    var countries = settings.flipNumberAndStreetCountries;
+    flipNumberAndStreetCountries = _.uniq(flipNumberAndStreetCountries.concat(countries));
+  }
+
   return applyLocalNamingConventions;
 }
 
@@ -12,11 +23,15 @@ function applyLocalNamingConventions(req, res, next) {
 
   // loop through data items and flip relevant number/street
   res.data.filter(function(place){
-    // only relevant for German addresses
-    if( 'DEU' !== place.alpha3 ){ return false; }
-    if( !place.hasOwnProperty('address') ){ return false; }
-    if( !place.address.hasOwnProperty('number') ){ return false; }
-    if( !place.address.hasOwnProperty('street') ){ return false; }
+    // relevant for some countries
+    var flip = place.parent.country_a.some(function(country) {
+      return _.includes(flipNumberAndStreetCountries, country);
+    });
+    if (!flip){ return false; }
+    if( !place.hasOwnProperty('address_parts') ){ return false; }
+    if( !place.address_parts.hasOwnProperty('number') ){ return false; }
+    if( !place.address_parts.hasOwnProperty('street') ){ return false; }
+
     return true;
   })
   .forEach( flipNumberAndStreet );
@@ -24,11 +39,11 @@ function applyLocalNamingConventions(req, res, next) {
   next();
 }
 
-// DE address should have the housenumber and street name flipped
+// flip the housenumber and street name
 // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101'
 function flipNumberAndStreet(place) {
-  var standard = ( place.address.number + ' ' + place.address.street ),
-      flipped  = ( place.address.street + ' ' + place.address.number );
+  var standard = ( place.address_parts.number + ' ' + place.address_parts.street ),
+      flipped  = ( place.address_parts.street + ' ' + place.address_parts.number );
 
   // flip street name and housenumber
   if( place.name.default === standard ){

middleware/parseBBox.js

@@ -0,0 +1,38 @@
+var logger = require('pelias-logger').get('api');
+
+/**
+ * Parses the bounding box property in docs, if one is found
+ */
+
+function setup() {
+  return function (req, res, next) {
+    // do nothing if no result data set
+    if (!res || !res.data) {
+      return next();
+    }
+
+    res.data = res.data.map(parseBBox);
+
+    next();
+  };
+}
+
+/*
+ * Parse the bbox property and form an object
+ */
+function parseBBox(place) {
+
+  if (place && place.bounding_box) {
+    try {
+      place.bounding_box = JSON.parse(place.bounding_box);
+    }
+    catch (err) {
+      logger.error('Invalid bounding_box json string:', place);
+      delete place.bounding_box;
+    }
+  }
+
+  return place;
+}
+
+module.exports = setup;

middleware/renamePlacenames.js

@@ -1,4 +1,4 @@
-var extend = require('extend');
+var _ = require('lodash');
 
 /**
  - P is a preferred English name
@@ -11,56 +11,73 @@ var extend = require('extend');
  - A is an abbreviation or code for the place (e.g. "NYC" for New
  York)
  */
-// config mapping of old names to new ones
-var NAME_MAP = {
+
+var ADDRESS_PROPS = {
   'number': 'housenumber',
   'zip': 'postalcode',
-  'alpha3': 'country_a',
-  'admin0': 'country',
-  'admin1': 'region',
-  'admin1_abbr': 'region_a',
-  'admin2': 'county',
-  'local_admin': 'localadmin',
-  'neighborhood': 'neighbourhood'
+  'street': 'street'
 };
 
-function setup() {
+var PARENT_PROPS = [
+  'country',
+  'country_id',
+  'country_a',
+  'macroregion',
+  'macroregion_id',
+  'macroregion_a',
+  'region',
+  'region_id',
+  'region_a',
+  'macrocounty',
+  'macrocounty_id',
+  'macrocounty_a',
+  'county',
+  'county_id',
+  'county_a',
+  'localadmin',
+  'localadmin_id',
+  'localadmin_a',
+  'locality',
+  'locality_id',
+  'locality_a',
+  'neighbourhood',
+  'neighbourhood_id'
+];
 
+
+function setup() {
   return renamePlacenames;
 }
 
 function renamePlacenames(req, res, next) {
-
   // do nothing if no result data set
   if (!res || !res.data) {
     return next();
   }
 
-  // loop through data items and remap placenames
-  res.data = res.data.map(renameProperties);
+  res.data = res.data.map(renameOneRecord);
 
   next();
 }
 
-function renameProperties(place) {
-  var newPlace = {};
-  Object.keys(place).forEach(function (property) {
-    if (property === 'address') {
-      extend(newPlace, renameProperties(place[property]));
-    }
-    else {
-      renameProperty(place, newPlace, property);
-    }
-  });
-  return newPlace;
-}
+/*
+ * Rename the fields in one record
+ */
+function renameOneRecord(place) {
+  if (place.address_parts) {
+    Object.keys(ADDRESS_PROPS).forEach(function (prop) {
+      place[ADDRESS_PROPS[prop]] = place.address_parts[prop];
+    });
+  }
 
-function renameProperty(oldObj, newObj, property) {
-  if (!oldObj.hasOwnProperty(property)) {
-    return;
+  // merge the parent block into the top level object to flatten the structure
+  if (place.parent) {
+    PARENT_PROPS.forEach(function (prop) {
+      place[prop] = place.parent[prop];
+    });
   }
 
-  newObj[(NAME_MAP[property] || property)] = oldObj[property];
+  return place;
 }
 
 module.exports = setup;

package.json

@@ -13,7 +13,9 @@
     "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao",
     "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
     "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
-    "docs": "rm -r docs; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs"
+    "docs": "./bin/generate-docs",
+    "lint": "jshint .",
+    "validate": "npm ls"
   },
   "repository": {
     "type": "git",
@@ -34,37 +36,42 @@
   },
   "dependencies": {
     "addressit": "git://github.com/dianashk/addressit.git#temp",
-    "async": "^0.9.0",
-    "check-types": "^3.3.1",
+    "async": "^1.5.2",
+    "check-types": "^6.0.0",
     "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",
+    "elasticsearch": "^10.1.3",
     "express": "^4.8.8",
     "express-http-proxy": "^0.6.0",
-    "extend": "2.0.1",
-    "geojson": "^0.2.1",
+    "extend": "3.0.0",
+    "geojson": "^0.3.0",
     "geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "geopipes-elasticsearch-backend": "^0.2.0",
     "iso3166-1": "^0.2.3",
-    "lodash": "^3.10.1",
+    "lodash": "^4.5.0",
     "markdown": "0.5.0",
-    "morgan": "1.5.2",
+    "morgan": "1.7.0",
     "pelias-config": "^1.0.1",
-    "pelias-esclient": "0.0.25",
     "pelias-logger": "^0.0.8",
-    "pelias-query": "^5.0.0",
-    "pelias-schema": "1.0.0",
-    "pelias-suggester-pipeline": "2.0.2",
-    "stats-lite": "^1.0.3",
-    "through2": "0.6.5"
+    "pelias-query": "6.2.0",
+    "pelias-suggester-pipeline": "2.0.4",
+    "stats-lite": "1.0.3",
+    "through2": "2.0.1"
   },
   "devDependencies": {
     "ciao": "^0.6.0",
-    "istanbul": "^0.3.13",
+    "difflet": "^1.0.1",
+    "istanbul": "^0.4.2",
     "jshint": "^2.5.6",
-    "nsp": "^0.3.0",
-    "precommit-hook": "^1.0.7",
+    "nsp": "^2.2.0",
+    "precommit-hook": "^3.0.0",
     "proxyquire": "^1.4.0",
-    "tap-dot": "^1.0.0",
-    "tape": "^2.13.4"
-  }
+    "tap-dot": "1.0.4",
+    "tape": "^4.4.0"
+  },
+  "pre-commit": [
+    "lint",
+    "validate",
+    "test"
+  ]
 }

public/attribution.md

@@ -3,5 +3,5 @@
 * Data from
    * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/)
    * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation
-   * [Quattroshapes](https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md) under [CC-BY-2.0](https://creativecommons.org/licenses/by/2.0/)
    * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/)
+   * [WhosOnFirst](http://whosonfirst.mapzen.com) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md)

query/autocomplete.js

@@ -27,14 +27,14 @@ query.score( peliasQuery.view.address('street') );
 query.score( peliasQuery.view.address('postcode') );
 
 // admin components
-query.score( peliasQuery.view.admin('alpha3') );
-query.score( peliasQuery.view.admin('admin0') );
-query.score( peliasQuery.view.admin('admin1') );
-query.score( peliasQuery.view.admin('admin1_abbr') );
-query.score( peliasQuery.view.admin('admin2') );
-query.score( peliasQuery.view.admin('local_admin') );
+query.score( peliasQuery.view.admin('country') );
+query.score( peliasQuery.view.admin('country_a') );
+query.score( peliasQuery.view.admin('region') );
+query.score( peliasQuery.view.admin('region_a') );
+query.score( peliasQuery.view.admin('county') );
+query.score( peliasQuery.view.admin('localadmin') );
 query.score( peliasQuery.view.admin('locality') );
-query.score( peliasQuery.view.admin('neighborhood') );
+query.score( peliasQuery.view.admin('neighbourhood') );
 
 // scoring boost
 query.score( views.focus_selected_layers( views.ngrams_strict ) );

query/autocomplete_defaults.js

@@ -30,57 +30,57 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'phrase:slop': 2,
 
   'focus:function': 'linear',
-  'focus:offset': '10km',
+  'focus:offset': '0km',
   'focus:scale': '250km',
   'focus:decay': 0.5,
-  'focus:weight': 3,
+  'focus:weight': 10,
 
   'function_score:score_mode': 'avg',
   'function_score:boost_mode': 'multiply',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 2000,
 
-  'admin:alpha3:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
-  'admin:alpha3:boost': 1000,
+  'admin:country_a:analyzer': 'standard',
+  'admin:country_a:field': 'parent.country_a',
+  'admin:country_a:boost': 1000,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
-  'admin:admin0:boost': 800,
+  'admin:country:analyzer': 'peliasAdmin',
+  'admin:country:field': 'parent.country',
+  'admin:country:boost': 800,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
-  'admin:admin1:boost': 600,
+  'admin:region:analyzer': 'peliasAdmin',
+  'admin:region:field': 'parent.region',
+  'admin:region:boost': 600,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
-  'admin:admin1_abbr:boost': 600,
+  'admin:region_a:analyzer': 'peliasAdmin',
+  'admin:region_a:field': 'parent.region_a',
+  'admin:region_a:boost': 600,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
-  'admin:admin2:boost': 400,
+  'admin:county:analyzer': 'peliasAdmin',
+  'admin:county:field': 'parent.county',
+  'admin:county:boost': 400,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
-  'admin:local_admin:boost': 200,
+  'admin:localadmin:analyzer': 'peliasAdmin',
+  'admin:localadmin:field': 'parent.localadmin',
+  'admin:localadmin:boost': 200,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 200,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
-  'admin:neighborhood:boost': 200,
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
+  'admin:neighbourhood:boost': 200,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',
@@ -90,6 +90,6 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'population:field': 'population',
   'population:modifier': 'log1p',
   'population:max_boost': 20,
-  'population:weight': 2
+  'population:weight': 3
 
 });

query/reverse_defaults.js

@@ -39,48 +39,48 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 3,
 
-  'admin:alpha3:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
-  'admin:alpha3:boost': 5,
+  'admin:country_a:analyzer': 'standard',
+  'admin:country_a:field': 'parent.country_a',
+  'admin:country_a:boost': 5,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
-  'admin:admin0:boost': 4,
+  'admin:country:analyzer': 'peliasAdmin',
+  'admin:country:field': 'parent.country',
+  'admin:country:boost': 4,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
-  'admin:admin1:boost': 3,
+  'admin:region:analyzer': 'peliasAdmin',
+  'admin:region:field': 'parent.region',
+  'admin:region:boost': 3,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
-  'admin:admin1_abbr:boost': 3,
+  'admin:region_a:analyzer': 'peliasAdmin',
+  'admin:region_a:field': 'parent.region_a',
+  'admin:region_a:boost': 3,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
-  'admin:admin2:boost': 2,
+  'admin:county:analyzer': 'peliasAdmin',
+  'admin:county:field': 'parent.county',
+  'admin:county:boost': 2,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
-  'admin:local_admin:boost': 1,
+  'admin:localadmin:analyzer': 'peliasAdmin',
+  'admin:localadmin:field': 'parent.localadmin',
+  'admin:localadmin:boost': 1,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 1,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
-  'admin:neighborhood:boost': 1,
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
+  'admin:neighbourhood:boost': 1,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',

query/search.js

@@ -25,19 +25,19 @@ query.score( peliasQuery.view.address('street') );
 query.score( peliasQuery.view.address('postcode') );
 
 // admin components
-query.score( peliasQuery.view.admin('alpha3') );
-query.score( peliasQuery.view.admin('admin0') );
-query.score( peliasQuery.view.admin('admin1') );
-query.score( peliasQuery.view.admin('admin1_abbr') );
-query.score( peliasQuery.view.admin('admin2') );
-query.score( peliasQuery.view.admin('local_admin') );
+query.score( peliasQuery.view.admin('country') );
+query.score( peliasQuery.view.admin('country_a') );
+query.score( peliasQuery.view.admin('region') );
+query.score( peliasQuery.view.admin('region_a') );
+query.score( peliasQuery.view.admin('county') );
+query.score( peliasQuery.view.admin('localadmin') );
 query.score( peliasQuery.view.admin('locality') );
-query.score( peliasQuery.view.admin('neighborhood') );
+query.score( peliasQuery.view.admin('neighbourhood') );
 
 // non-scoring hard filters
 query.filter( peliasQuery.view.boundary_circle );
 query.filter( peliasQuery.view.boundary_rect );
-
+query.filter( peliasQuery.view.sources );
 // --------------------------------
 
 /**
@@ -51,6 +51,8 @@ function generateQuery( clean ){
   // input text
   vs.var( 'input:name', clean.text );
 
+  vs.var( 'sources', clean.sources);
+
   // size
   if( clean.querySize ) {
     vs.var( 'size', clean.querySize );

query/search_defaults.js

@@ -39,48 +39,48 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
-  'address:housenumber:field': 'address.number',
+  'address:housenumber:field': 'address_parts.number',
   'address:housenumber:boost': 2,
 
   'address:street:analyzer': 'peliasStreet',
-  'address:street:field': 'address.street',
+  'address:street:field': 'address_parts.street',
   'address:street:boost': 5,
 
   'address:postcode:analyzer': 'peliasZip',
-  'address:postcode:field': 'address.zip',
+  'address:postcode:field': 'address_parts.zip',
   'address:postcode:boost': 20,
 
-  'admin:alpha3:analyzer': 'standard',
-  'admin:alpha3:field': 'alpha3',
-  'admin:alpha3:boost': 5,
+  'admin:country_a:analyzer': 'standard',
+  'admin:country_a:field': 'parent.country_a',
+  'admin:country_a:boost': 5,
 
-  'admin:admin0:analyzer': 'peliasAdmin',
-  'admin:admin0:field': 'admin0',
-  'admin:admin0:boost': 4,
+  'admin:country:analyzer': 'peliasAdmin',
+  'admin:country:field': 'parent.country',
+  'admin:country:boost': 4,
 
-  'admin:admin1:analyzer': 'peliasAdmin',
-  'admin:admin1:field': 'admin1',
-  'admin:admin1:boost': 3,
+  'admin:region:analyzer': 'peliasAdmin',
+  'admin:region:field': 'parent.region',
+  'admin:region:boost': 3,
 
-  'admin:admin1_abbr:analyzer': 'peliasAdmin',
-  'admin:admin1_abbr:field': 'admin1_abbr',
-  'admin:admin1_abbr:boost': 3,
+  'admin:region_a:analyzer': 'peliasAdmin',
+  'admin:region_a:field': 'parent.region_a',
+  'admin:region_a:boost': 3,
 
-  'admin:admin2:analyzer': 'peliasAdmin',
-  'admin:admin2:field': 'admin2',
-  'admin:admin2:boost': 2,
+  'admin:county:analyzer': 'peliasAdmin',
+  'admin:county:field': 'parent.county',
+  'admin:county:boost': 2,
 
-  'admin:local_admin:analyzer': 'peliasAdmin',
-  'admin:local_admin:field': 'local_admin',
-  'admin:local_admin:boost': 1,
+  'admin:localadmin:analyzer': 'peliasAdmin',
+  'admin:localadmin:field': 'parent.localadmin',
+  'admin:localadmin:boost': 1,
 
   'admin:locality:analyzer': 'peliasAdmin',
-  'admin:locality:field': 'locality',
+  'admin:locality:field': 'parent.locality',
   'admin:locality:boost': 1,
 
-  'admin:neighborhood:analyzer': 'peliasAdmin',
-  'admin:neighborhood:field': 'neighborhood',
-  'admin:neighborhood:boost': 1,
+  'admin:neighbourhood:analyzer': 'peliasAdmin',
+  'admin:neighbourhood:field': 'parent.neighbourhood',
+  'admin:neighbourhood:boost': 1,
 
   'popularity:field': 'popularity',
   'popularity:modifier': 'log1p',

query/text_parser.js

@@ -1,6 +1,20 @@
 
 var logger = require('pelias-logger').get('api');
-var adminFields = require('../helper/adminFields')();
+
+/*
+This list should only contain admin fields we are comfortable matching in the case
+when we can't identify parts of an address. This shouldn't contain fields like country_a
+or postalcode because we should only try to match those when we're sure that's what they are.
+ */
+var adminFields = [
+  'country',
+  'region',
+  'region_a',
+  'county',
+  'localadmin',
+  'locality',
+  'neighbourhood'
+];
 
 /**
   @todo: refactor me
@@ -48,17 +62,17 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
 
   // city
   if( parsed_text.hasOwnProperty('city') ){
-    vs.var( 'input:admin2', parsed_text.city );
+    vs.var( 'input:county', parsed_text.city );
   }
 
   // state
   if( parsed_text.hasOwnProperty('state') ){
-    vs.var( 'input:admin1_abbr', parsed_text.state );
+    vs.var( 'input:region_a', parsed_text.state );
   }
 
   // country
   if( parsed_text.hasOwnProperty('country') ){
-    vs.var( 'input:alpha3', parsed_text.country );
+    vs.var( 'input:country_a', parsed_text.country );
   }
 
   // ==== deal with the 'leftover' components ====
@@ -76,11 +90,10 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
   // if we have 'leftovers' then assign them to any fields which
   // currently don't have a value assigned.
   if( leftoversString.length ){
-    var unmatchedAdminFields = adminFields.slice();
-    
+
     // cycle through fields and set fields which
     // are still currently unset
-    unmatchedAdminFields.forEach( function( key ){
+    adminFields.forEach( function( key ){
       if( !vs.isset( 'input:' + key ) ){
         vs.var( 'input:' + key, leftoversString );
       }

query/view/focus_selected_layers.js

@@ -22,10 +22,8 @@ module.exports = function( subview ){
     if( view && view.hasOwnProperty('function_score') ){
       view.function_score.filter = {
         'or': [
-          { 'type': { 'value': 'osmnode' } },
-          { 'type': { 'value': 'osmway' } },
-          { 'type': { 'value': 'osmaddress' } },
-          { 'type': { 'value': 'openaddresses' } }
+          { 'term': { 'layer': 'venue' } },
+          { 'term': { 'layer': 'address' } }
         ]
       };
     }

routes/v1.js

@@ -12,7 +12,6 @@ var sanitisers = {
 
 /** ----------------------- middleware ------------------------ **/
 var middleware = {
-  types: require('../middleware/_types'),
   calcSize: require('../middleware/sizeCalculator')
 };
 
@@ -35,7 +34,8 @@ var postProc = {
   localNamingConventions: require('../middleware/localNamingConventions'),
   renamePlacenames: require('../middleware/renamePlacenames'),
   geocodeJSON: require('../middleware/geocodeJSON'),
-  sendJSON: require('../middleware/sendJSON')
+  sendJSON: require('../middleware/sendJSON'),
+  parseBoundingBox: require('../middleware/parseBBox')
 };
 
 /**
@@ -59,7 +59,6 @@ function addRoutes(app, peliasConfig) {
     ]),
     search: createRouter([
       sanitisers.search.middleware,
-      middleware.types,
       middleware.calcSize(),
       controllers.search(),
       postProc.distances('focus.point.'),
@@ -67,24 +66,24 @@ function addRoutes(app, peliasConfig) {
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
     autocomplete: createRouter([
       sanitisers.autocomplete.middleware,
-      middleware.types,
       controllers.search(null, require('../query/autocomplete')),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
     reverse: createRouter([
       sanitisers.reverse.middleware,
-      middleware.types,
       middleware.calcSize(),
       controllers.search(undefined, reverseQuery),
       postProc.distances('point.'),
@@ -94,6 +93,7 @@ function addRoutes(app, peliasConfig) {
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
@@ -102,6 +102,7 @@ function addRoutes(app, peliasConfig) {
       controllers.place(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),

sanitiser/_boundary_country.js

@@ -13,7 +13,7 @@ function sanitize(raw, clean) {
   if (check.assigned(country)){
 
     // must be valid string
-    if (!check.unemptyString(country)) {
+    if (!check.nonEmptyString(country)) {
       messages.errors.push('boundary.country is not a string');
     }
 

sanitiser/_categories.js

@@ -11,7 +11,7 @@ function sanitize( raw, clean ){
   clean.categories = [];
 
   // if categories string has been set
-  if( check.unemptyString( raw.categories ) ){
+  if( check.nonEmptyString( raw.categories ) ){
 
     // map input categories to valid format
     clean.categories = raw.categories.split(',')

sanitiser/_details.js

@@ -22,7 +22,7 @@ function sanitize( raw, clean ){
 // be lenient with 'truthy' values
 function isTruthy(val) {
   if( check.string( val ) ){
-    return _.contains( ['true', '1'], val );
+    return _.includes( ['true', '1'], val );
   }
 
   return val === 1 || val === true;

sanitiser/_flag_bool.js

@@ -44,7 +44,7 @@ function sanitize( raw, clean, opts ){
  * @returns {boolean}
  */
 function isTruthy(val) {
-  return _.contains( ['true', '1', 1, true], val );
+  return _.includes( ['true', '1', 1, true], val );
 }
 
 module.exports = setup;

sanitiser/_groups.js

@@ -10,7 +10,7 @@ var _ = require('lodash');
  * returns true if all are present, false if none are present, throws an exception otherwise
  */
 function optional_group(object, keys) {
-  var contained_in_object = _.contains.bind(null, Object.keys(object));
+  var contained_in_object = _.includes.bind(null, Object.keys(object));
 
   if (keys.every(contained_in_object)) {
     return true;
@@ -26,7 +26,7 @@ function optional_group(object, keys) {
  * An error will be thrown if any of the keys are missing from the object
  */
 function required_group(object, keys) {
-  var contained_in_object = _.contains.bind(null, Object.keys(object));
+  var contained_in_object = _.includes.bind(null, Object.keys(object));
 
   if (keys.every(contained_in_object)) {
     return true;

sanitiser/_ids.js

@@ -30,32 +30,25 @@ function sanitizeId(rawId, messages) {
   var id = parts.slice(2).join(ID_DELIM);
 
   // check if any parts of the gid are empty
-  if (_.contains([source, layer, id], '')) {
+  if (_.includes([source, layer, id], '')) {
     messages.errors.push( formatError(rawId) );
     return;
   }
 
-  if (!_.contains(type_mapping.sources, source)) {
+  if (!_.includes(type_mapping.sources, source)) {
     messages.errors.push( targetError(source, type_mapping.sources) );
     return;
   }
 
-  if (!_.contains(type_mapping.layers, layer)) {
+  if (!_.includes(type_mapping.layers, layer)) {
     messages.errors.push( targetError(layer, type_mapping.layers) );
     return;
   }
 
-  //TODO: remove this once we have a better set of layers for Geonames
-  var types;
-  if (source === 'gn' || source === 'geonames') {
-    types = ['geoname'];
-  } else {
-    types = type_mapping.source_and_layer_to_type(source, layer);
-  }
-
   return {
+    source: source,
+    layer: layer,
     id: id,
-    types: types
   };
 }
 
@@ -63,7 +56,7 @@ function sanitize( raw, clean ){
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
-  if (!check.unemptyString( raw.ids )) {
+  if (!check.nonEmptyString( raw.ids )) {
     messages.errors.push( lengthError);
     return messages;
   }
@@ -72,10 +65,10 @@ function sanitize( raw, clean ){
   var rawIds = raw.ids.split(',');
 
   // deduplicate
-  rawIds = _.unique(rawIds);
+  rawIds = _.uniq(rawIds);
 
   // ensure all elements are valid non-empty strings
-  if (!rawIds.every(check.unemptyString)) {
+  if (!rawIds.every(check.nonEmptyString)) {
       messages.errors.push( lengthError );
   }
 

sanitiser/_source.js

@@ -1,46 +0,0 @@
-
-var _ = require('lodash'),
-    check = require('check-types'),
-    sources_map = require( '../query/sources' );
-
-var ALL_SOURCES = Object.keys(sources_map),
-    ALL_SOURCES_JOINED = ALL_SOURCES.join(',');
-
-function sanitize( raw, clean ) {
-
-  // error & warning messages
-  var messages = { errors: [], warnings: [] };
-
-  // init clean.types (if not already init)
-  clean.types = clean.types || {};
-
-  // default case (no layers specified in GET params)
-  // don't even set the from_layers key in this case
-  if( check.unemptyString( raw.source ) ){
-
-    var sources = raw.source.split(',');
-
-    var invalid_sources = sources.filter(function(source) {
-      return !_.contains( ALL_SOURCES, source );
-    });
-
-    if( invalid_sources.length > 0 ){
-      invalid_sources.forEach( function( invalid ){
-        messages.errors.push('\'' + invalid + '\' is an invalid source parameter. Valid options: ' + ALL_SOURCES_JOINED);
-      });
-    }
-
-    else {
-      var types = sources.reduce(function(acc, source) {
-        return acc.concat(sources_map[source]);
-      }, []);
-
-      clean.types.from_source = types;
-    }
-
-  }
-
-  return messages;
-}
-
-module.exports = sanitize;

sanitiser/_sources_and_layers.js

@@ -0,0 +1,37 @@
+var _ = require( 'lodash' );
+var type_mapping = require( '../helper/type_mapping' );
+
+/*
+ * This sanitiser depends on clean.layers and clean.sources
+ * so it has to be run after those sanitisers have been run
+ */
+function sanitize( raw, clean ){
+  var messages = { errors: [], warnings: [] };
+
+  var possible_errors = [];
+  var at_least_one_valid_combination = false;
+
+  if (clean.layers && clean.sources) {
+    clean.sources.forEach(function(source) {
+      var layers_for_source = type_mapping.layers_by_source[source];
+      clean.layers.forEach(function(layer) {
+        if (_.includes(layers_for_source, layer)) {
+          at_least_one_valid_combination = true;
+        } else {
+          var message = 'You have specified both the `sources` and `layers` ' +
+            'parameters in a combination that will return no results: the ' +
+            source + ' source has nothing in the ' + layer + ' layer';
+          possible_errors.push(message);
+        }
+      });
+    });
+
+    if (!at_least_one_valid_combination) {
+      messages.errors = possible_errors;
+    }
+  }
+
+  return messages;
+}
+
+module.exports = sanitize;

sanitiser/_targets.js

@@ -1,34 +1,33 @@
-
 var _ = require('lodash'),
     check = require('check-types');
 
+function getValidKeys(mapping) {
+  return _.uniq(Object.keys(mapping)).join(',');
+}
+
 function setup( paramName, targetMap ) {
   return function( raw, clean ){
     return sanitize( raw, clean, {
       paramName: paramName,
       targetMap: targetMap,
-      targetMapKeysString: Object.keys(targetMap).join(',')
+      targetMapKeysString: getValidKeys(targetMap)
     });
   };
 }
 
 function sanitize( raw, clean, opts ) {
-
   // error & warning messages
   var messages = { errors: [], warnings: [] };
 
-  // init clean.types
-  clean.types = clean.types || {};
-
   // the string of targets (comma delimeted)
   var targetsString = raw[opts.paramName];
 
   // trim whitespace
-  if( check.unemptyString( targetsString ) ){
+  if( check.nonEmptyString( targetsString ) ){
     targetsString = targetsString.trim();
 
     // param must be a valid non-empty string
-    if( !check.unemptyString( targetsString ) ){
+    if( !check.nonEmptyString( targetsString ) ){
       messages.errors.push(
         opts.paramName + ' parameter cannot be an empty string. Valid options: ' + opts.targetMapKeysString
       );
@@ -51,19 +50,13 @@ function sanitize( raw, clean, opts ) {
 
       // only set types value when no error occured
       if( !messages.errors.length ){
-
-        // store the values under a new key as 'clean.types.from_*'
-        var typesKey = 'from_' + opts.paramName;
-
-        // ?
-        clean.types[typesKey] = targets.reduce(function(acc, target) {
+        clean[opts.paramName] = targets.reduce(function(acc, target) {
           return acc.concat(opts.targetMap[target]);
         }, []);
 
         // dedupe in case aliases expanded to common things or user typed in duplicates
-        clean.types[typesKey] = _.unique(clean.types[typesKey]);
+        clean[opts.paramName] = _.uniq(clean[opts.paramName]);
       }
-
     }
   }
 

sanitiser/_text.js

@@ -8,7 +8,7 @@ function sanitize( raw, clean ){
   var messages = { errors: [], warnings: [] };
 
   // invalid input 'text'
-  if( !check.unemptyString( raw.text ) ){
+  if( !check.nonEmptyString( raw.text ) ){
     messages.errors.push('invalid param \'text\': text length, must be >0');
   }
 
@@ -23,10 +23,6 @@ function sanitize( raw, clean ){
     if (check.assigned(parsed_text)) {
       clean.parsed_text = parsed_text;
     }
-
-    // try to set layers from query parser results
-    clean.types = clean.layers || {};
-    clean.types.from_text_parser = text_parser.get_layers(clean.text);
   }
 
   return messages;

sanitiser/reverse.js

@@ -3,8 +3,10 @@ var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
-      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
-      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
+      // depends on the layers and sources sanitisers, must be run after them
+      sources_and_layers: require('../sanitiser/_sources_and_layers'),
       quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(),
       size: require('../sanitiser/_size'),
       private: require('../sanitiser/_flag_bool')('private', false),

sanitiser/search.js

@@ -5,8 +5,10 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
       size: require('../sanitiser/_size'),
-      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type),
-      sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
+      // depends on the layers and sources sanitisers, must be run after them
+      sources_and_layers: require('../sanitiser/_sources_and_layers'),
       quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),

service/mget.js

@@ -24,7 +24,6 @@ function service( backend, query, cb ){
 
   // query new backend
   backend().client.mget( cmd, function( err, data ){
-
     // handle backend errors
     if( err ){ return cb( err ); }
 

src/backend.js

@@ -1,6 +1,6 @@
-
+var config = require( 'pelias-config' ).generate().esclient;
 var Backend = require('geopipes-elasticsearch-backend'),
-    client = require('pelias-esclient')(),
+    client = require('elasticsearch').Client(config),
     backends = {};
 
 function getBackend( index, type ){
@@ -11,4 +11,4 @@ function getBackend( index, type ){
   return backends[key];
 }
 
-module.exports = getBackend;
+module.exports = getBackend;

test/ciao/place/basic_place.coffee

@@ -29,5 +29,5 @@ should.not.exist json.geocoding.errors
 should.not.exist json.geocoding.warnings
 
 #? inputs
-json.geocoding.query['ids'].should.eql [{ id: '1', types: [ 'geoname' ] }]
+json.geocoding.query['ids'].should.eql [{ id: '1', layer: 'venue', source: 'geonames' }]
 should.not.exist json.geocoding.query['size']

test/ciao/reverse/layers_alias_coarse.coffee

@@ -30,5 +30,16 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
-json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.layers.should.eql [ "continent",
+  "macrocountry",
+  "country",
+  "dependency",
+  "region",
+  "locality",
+  "localadmin",
+  "county",
+  "macrohood",
+  "neighbourhood",
+  "microhood",
+  "disputed"
+]

test/ciao/reverse/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/layers_multiple.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]
+json.geocoding.query.layers.should.eql ["country","region"]

test/ciao/reverse/layers_single.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
-json.geocoding.query['type'].should.eql ["admin0"]
+json.geocoding.query.layers.should.eql ["country"]

test/ciao/reverse/sources_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ]
+json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings

test/ciao/reverse/sources_layers_invalid_combo.coffee

@@ -1,6 +1,6 @@
 
 #> sources and layers specified (invalid combo)
-path: '/v1/reverse?point.lat=1&point.lon=2&sources=quattroshapes&layers=address'
+path: '/v1/reverse?point.lat=1&point.lon=2&sources=whosonfirst&layers=address'
 
 #? 200 ok
 response.statusCode.should.be.equal 400
@@ -24,13 +24,12 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ]
+json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ]
 
 #? expected warnings
 json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ]
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
-json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
-should.not.exist json.geocoding.query['type']
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["whosonfirst"]

test/ciao/reverse/sources_layers_valid_combo.coffee

@@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
-json.geocoding.query['type'].should.eql ["openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["openaddresses"]

test/ciao/reverse/sources_multiple.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"]
-json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"]
+json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"]

test/ciao/reverse/sources_single.coffee

@@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings
 
 #? inputs
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"]
-json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"]
+json.geocoding.query.sources.should.eql ["openstreetmap"]

test/ciao/search/layers_alias_address.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
-json.geocoding.query['type'].should.eql ["osmaddress","openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]

test/ciao/search/layers_alias_coarse.coffee

@@ -31,5 +31,16 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
-json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.layers.should.eql [ "continent",
+  "macrocountry",
+  "country",
+  "dependency",
+  "region",
+  "locality",
+  "localadmin",
+  "county",
+  "macrohood",
+  "neighbourhood",
+  "microhood",
+  "disputed"
+]

test/ciao/search/layers_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
-should.not.exist json.geocoding.query['type']

test/ciao/search/layers_mix_invalid_valid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ]
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
-should.not.exist json.geocoding.query['type']
+should.not.exist json.geocoding.query['layers']

test/ciao/search/layers_multiple.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"]
-json.geocoding.query['type'].should.eql ["admin0","admin1"]
+json.geocoding.query.layers.should.eql ["country","region"]

test/ciao/search/layers_single.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["admin0"]
-json.geocoding.query['type'].should.eql ["admin0"]
+json.geocoding.query.layers.should.eql ["country"]

test/ciao/search/sources_invalid.coffee

@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ]
+json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ]
 
 #? expected warnings
 should.not.exist json.geocoding.warnings
@@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-should.not.exist json.geocoding.query['types']
-should.not.exist json.geocoding.query['type']

test/ciao/search/sources_layers_invalid_combo.coffee

@@ -1,6 +1,6 @@
 
 #> sources and layers specified (invalid combo)
-path: '/v1/search?text=a&sources=quattroshapes&layers=address'
+path: '/v1/search?text=a&sources=whosonfirst&layers=address'
 
 #? 200 ok
 response.statusCode.should.be.equal 400
@@ -24,7 +24,7 @@ json.features.should.be.instanceof Array
 
 #? expected errors
 should.exist json.geocoding.errors
-json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ]
+json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ]
 
 #? expected warnings
 json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ]
@@ -32,6 +32,6 @@ json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data sour
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
-json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"]
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["whosonfirst"]
 should.not.exist json.geocoding.query['type']

test/ciao/search/sources_layers_valid_combo.coffee

@@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"]
-json.geocoding.query['type'].should.eql ["openaddresses"]
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["openaddresses"]

test/ciao/search/sources_multiple.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"]
-json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"]
+json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"]

test/ciao/search/sources_single.coffee

@@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings
 #? inputs
 json.geocoding.query['text'].should.eql 'a'
 json.geocoding.query['size'].should.eql 10
-json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"]
-json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"]
+json.geocoding.query.sources.should.eql ["openstreetmap"]

test/unit/controller/place.js

@@ -41,7 +41,7 @@ module.exports.tests.functional_success = function(test, common) {
 
   test('functional success', function(t) {
     var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
     var res = {
@@ -57,7 +57,7 @@ module.exports.tests.functional_success = function(test, common) {
         t.deepEqual(json.features, expected, 'values correctly mapped');
       }
     };
-    var req = { clean: { ids: [ {'id' : 123, types: [ 'a' ] } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, layers: [ 'a' ] } ] }, errors: [], warnings: [] };
     var next = function next() {
       t.equal(req.errors.length, 0, 'next was called without error');
       t.end();
@@ -70,10 +70,10 @@ module.exports.tests.functional_success = function(test, common) {
 module.exports.tests.functional_failure = function(test, common) {
   test('functional failure', function(t) {
     var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
-      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command');
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: ['b'] } ] } }, 'correct backend command');
     });
     var controller = setup( backend );
-    var req = { clean: { ids: [ {'id' : 123, types: [ 'b' ] } ] }, errors: [], warnings: [] };
+    var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] };
     var next = function( message ){
       t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler');
       t.end();

test/unit/controller/search.js

@@ -50,9 +50,11 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid1',
       _score: 10,
       _type: 'mytype1',
-      admin0: 'country1',
-      admin1: 'state1',
-      admin2: 'city1',
+      parent: {
+        country: ['country1'],
+        region: ['state1'],
+        county: ['city1']
+      },
       center_point: { lat: 100.1, lon: -50.5 },
       name: { default: 'test name1' },
       value: 1
@@ -61,9 +63,11 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid2',
       _score: 20,
       _type: 'mytype2',
-      admin0: 'country2',
-      admin1: 'state2',
-      admin2: 'city2',
+      parent: {
+        country: ['country2'],
+        region: ['state2'],
+        county: ['city2']
+      },
       center_point: { lat: 100.2, lon: -51.5 },
       name: { default: 'test name2' },
       value: 2

test/unit/fixture/autocomplete_linguistic_final_token.js

@@ -31,15 +31,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -60,17 +56,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -35,35 +35,25 @@ module.exports = {
                       'lat': 29.49136,
                       'lon': -82.50622
                     },
-                    'offset': '10km',
+                    'offset': '0km',
                     'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 3
+                'weight': 10
               }],
               'score_mode': 'avg',
               'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
-                    'type': {
-                      'value': 'osmnode'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmway'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmaddress'
+                    'term': {
+                      'layer': 'venue'
                     }
                   },
                   {
-                    'type': {
-                      'value': 'openaddresses'
+                    'term': {
+                      'layer': 'address'
                     }
                   }
                 ]
@@ -85,15 +75,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -114,17 +100,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -35,35 +35,25 @@ module.exports = {
                       'lat': 0,
                       'lon': 0
                     },
-                    'offset': '10km',
+                    'offset': '0km',
                     'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 3
+                'weight': 10
               }],
               'score_mode': 'avg',
               'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
-                    'type': {
-                      'value': 'osmnode'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmway'
-                    }
-                  },
-                  {
-                    'type': {
-                      'value': 'osmaddress'
+                    'term': {
+                      'layer': 'venue'
                     }
                   },
                   {
-                    'type': {
-                      'value': 'openaddresses'
+                    'term': {
+                      'layer': 'address'
                     }
                   }
                 ]
@@ -85,15 +75,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -114,17 +100,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_multiple_tokens.js

@@ -42,15 +42,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -71,17 +67,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_only.js

@@ -31,15 +31,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -60,17 +56,13 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
-                'weight': 2
+                'weight': 3
               }]
             }
           }]

test/unit/fixture/autocomplete_linguistic_with_admin.js

@@ -20,7 +20,7 @@ module.exports = {
           'should': [
             {
               'match': {
-                'admin0': {
+                'parent.country': {
                   'analyzer': 'peliasAdmin',
                   'boost': 800,
                   'query': 'three'
@@ -29,7 +29,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin1': {
+                'parent.region': {
                   'analyzer': 'peliasAdmin',
                   'boost': 600,
                   'query': 'three'
@@ -38,7 +38,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin1_abbr': {
+                'parent.region_a': {
                   'analyzer': 'peliasAdmin',
                   'boost': 600,
                   'query': 'three'
@@ -47,7 +47,7 @@ module.exports = {
             },
             {
               'match': {
-                'admin2': {
+                'parent.county': {
                   'analyzer': 'peliasAdmin',
                   'boost': 400,
                   'query': 'three'
@@ -56,7 +56,7 @@ module.exports = {
             },
             {
               'match': {
-                'local_admin': {
+                'parent.localadmin': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -65,7 +65,7 @@ module.exports = {
             },
             {
               'match': {
-                'locality': {
+                'parent.locality': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -74,7 +74,7 @@ module.exports = {
             },
             {
               'match': {
-                'neighborhood': {
+                'parent.neighbourhood': {
                   'analyzer': 'peliasAdmin',
                   'boost': 200,
                   'query': 'three'
@@ -99,18 +99,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             },
             {
@@ -131,18 +127,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
-                    'weight': 2
+                    'weight': 3
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             }
           ]

test/unit/fixture/dedupe_elasticsearch_nonascii_results.js

@@ -7,9 +7,11 @@ module.exports = [
     'name': {
       'default': '万里长城万里长城'
     },
-    'country_a': 'CHN',
-    'country': 'China',
-    'region': 'Beijing',
+    'parent': {
+      'country_a': ['CHN'],
+      'country': ['China'],
+      'region': ['Beijing']
+    },
     'confidence': 0.733
   },
   {
@@ -20,9 +22,11 @@ module.exports = [
     'name': {
       'default': '万里长城'
     },
-    'country_a': 'CHN',
-    'country': 'China',
-    'region': 'Beijing',
+    'parent': {
+      'country_a': ['CHN'],
+      'country': ['China'],
+      'region': ['Beijing']
+    },
     'confidence': 0.733
   },
   {
@@ -33,12 +37,14 @@ module.exports = [
     'name': {
       'default': '万里花'
     },
-    'country_a': 'JPN',
-    'country': 'Japan',
-    'region': 'Tokyo',
-    'county': '豊島区',
-    'locality': 'Tokyo',
-    'neighbourhood': '２丁目',
+    'parent': {
+      'country_a': ['JPN'],
+      'country': ['Japan'],
+      'region': ['Tokyo'],
+      'county': ['豊島区'],
+      'locality': ['Tokyo'],
+      'neighbourhood': ['２丁目']
+    },
     'confidence': 0.646
   },
   {
@@ -49,13 +55,15 @@ module.exports = [
     'name': {
       'default': '万里加油站'
     },
-    'address': {
+    'address_parts': {
       'street': 'S308',
       'postalcode': '312044'
     },
-    'country_a': 'CHN',
-    'country': 'China',
-    'region': 'Zhejiang',
+    'parent': {
+      'country_a': ['CHN'],
+      'country': ['China'],
+      'region': ['Zhejiang']
+    },
     'confidence': 0.646
   }
 ];

test/unit/fixture/dedupe_elasticsearch_results.js

@@ -4,23 +4,79 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.039265
     },
+    'address_parts': {},
+    'parent': {
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
+    'name': {
+      'default': 'East Lampeter High School'
+    },
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'venue',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  { // same as above, but change the neighbourhood
+    'center_point': {
+      'lon': -77.207456,
+      'lat': 41.039265
+    },
+    'address': {},
+    'parent': {
+      'localadmin': 'East Lampeter',
+      'region_a': 'PA',
+      'region': 'Pennsylvania',
+      'locality': 'Smoketown',
+      'country_a': 'USA',
+      'county': 'Lancaster County',
+      'country': 'United States',
+      'neighbourhood': 'Blueland' // ###
+    },
+    'name': {
+      'default': 'East Lampeter High School'
+    },
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'venue',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  { // same as #1, but change the locality
+    'center_point': {
+      'lon': -73.207456,
+      'lat': 42.039265
+    },
     'address': {},
-    'local_admin': 'East Lampeter',
-    'admin1_abbr': 'PA',
+    'parent': {
+      'localadmin': 'East Lampeter',
+      'region_a': 'PA',
+      'region': 'Pennsylvania',
+      'locality': 'Firetown', // ###
+      'country_a': 'USA',
+      'county': 'Lancaster County',
+      'country': 'United States',
+      'neighbourhood': 'Greenland'
+    },
     'name': {
       'default': 'East Lampeter High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
     '_id': '357321757',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -29,23 +85,25 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.039265
     },
-    'address': {},
-    'local_admin': 'East Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
     'name': {
       'default': 'East Lampeter, High-School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
     '_id': '357321757',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -54,18 +112,20 @@ module.exports = [
       'lon': -76.23246,
       'lat': 39.99288
     },
-    'address': {},
-    'local_admin': 'West Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'localadmin': ['West Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Lampeter'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Wheatland Mills']
+    },
     'name': {
       'default': 'Lampeter-Strasburg High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Lampeter',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Wheatland Mills',
     'category': [
       'education'
     ],
@@ -79,18 +139,20 @@ module.exports = [
       'lon': -76.20746,
       'lat': 40.03927
     },
-    'address': {},
-    'local_admin': 'East Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
     'name': {
       'default': 'East Lampeter High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
@@ -104,23 +166,25 @@ module.exports = [
       'lon': -76.232457,
       'lat': 39.992877
     },
-    'address': {},
-    'local_admin': 'West Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'region': ['Pennsylvania'],
+      'locality': ['Lampeter'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Wheatland Mills'],
+      'localadmin': ['West Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'Lampeter-Strasburg High School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Lampeter',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Wheatland Mills',
     'category': [
       'education'
     ],
     '_id': '357294404',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.2367082,
     'confidence': 0.879
   },
@@ -129,23 +193,25 @@ module.exports = [
       'lon': -76.207456,
       'lat': 40.038987
     },
-    'address': {},
-    'local_admin': 'East Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland'],
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'East Lampeter School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
     '_id': '357283977',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.1036991,
     'confidence': 0.664
   },
@@ -154,18 +220,20 @@ module.exports = [
       'lon': -76.20746,
       'lat': 40.03899
     },
-    'address': {},
-    'local_admin': 'East Lampeter',
-    'admin1_abbr': 'PA',
+    'address_parts': {},
+    'parent': {
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland'],
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA']
+    },
     'name': {
       'default': 'East Lampeter School'
     },
-    'admin1': 'Pennsylvania',
-    'locality': 'Smoketown',
-    'alpha3': 'USA',
-    'admin2': 'Lancaster County',
-    'admin0': 'United States',
-    'neighborhood': 'Greenland',
     'category': [
       'education'
     ],
@@ -179,22 +247,24 @@ module.exports = [
       'lon': -94.167445,
       'lat': 38.762788
     },
-    'address': {},
-    'local_admin': 'Polk',
-    'admin1_abbr': 'MO',
+    'address_parts': {},
+    'parent': {
+      'region': ['Missouri'],
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Cass County'],
+      'country': ['United States'],
+      'localadmin': ['Polk'],
+      'region_a': ['MO']
+    },
     'name': {
       'default': 'Strasburg School'
     },
-    'admin1': 'Missouri',
-    'locality': 'Strasburg',
-    'alpha3': 'USA',
-    'admin2': 'Cass County',
-    'admin0': 'United States',
     'category': [
       'education'
     ],
     '_id': '358058986',
-    '_type': 'osmnode',
+    '_type': 'venue',
     '_score': 1.0492544,
     'confidence': 0.658
   },
@@ -203,17 +273,19 @@ module.exports = [
       'lon': -78.36317,
       'lat': 38.98445
     },
-    'address': {},
-    'admin1_abbr': 'VA',
+    'address_parts': {},
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'Virginia',
-    'locality': 'Strasburg',
-    'alpha3': 'USA',
-    'admin2': 'Shenandoah County',
-    'admin0': 'United States',
-    'neighborhood': 'Strasburg Junction',
+    'parent': {
+      'region_a': ['VA'],
+      'region': ['Virginia'],
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Shenandoah County'],
+      'country': ['United States'],
+      'neighbourhood': ['Strasburg Junction']
+    },
     'category': [
       'education'
     ],
@@ -227,17 +299,19 @@ module.exports = [
       'lon': -100.16516,
       'lat': 46.13427
     },
-    'address': {},
-    'local_admin': 'Strasburg',
-    'admin1_abbr': 'ND',
+    'address_parts': {},
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'North Dakota',
-    'locality': 'Strasburg',
-    'alpha3': 'USA',
-    'admin2': 'Emmons County',
-    'admin0': 'United States',
+    'parent': {
+      'localadmin': ['Strasburg'],
+      'region_a': ['ND'],
+      'region': ['North Dakota'],
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Emmons County'],
+      'country': ['United States']
+    },
     'category': [
       'education'
     ],
@@ -251,23 +325,25 @@ module.exports = [
       'lon': -81.532392,
       'lat': 40.597578
     },
-    'address': {},
-    'local_admin': 'Franklin',
-    'admin1_abbr': 'OH',
+    'address_parts': {},
     'name': {
       'default': 'Strasburg High School'
     },
-    'admin1': 'Ohio',
-    'locality': 'Strasburg',
-    'alpha3': 'USA',
-    'admin2': 'Tuscarawas County',
-    'admin0': 'United States',
+    'parent': {
+      'localadmin': ['Franklin'],
+      'region_a': ['OH'],
+      'region': ['Ohio'],
+      'locality': ['Strasburg'],
+      'country_a': ['USA'],
+      'county': ['Tuscarawas County'],
+      'country': ['United States']
+    },
     'category': [
       'education'
     ],
     '_id': '356646971',
-    '_type': 'osmway',
+    '_type': 'venue',
     '_score': 0.9724125,
     'confidence': 0.649
   }
-];
+];

test/unit/fixture/reverse_with_boundary_country.js

@@ -8,7 +8,7 @@ module.exports = {
           'must': [
             {
               'match': {
-                'alpha3': {
+                'parent.country_a': {
                   'analyzer': 'standard',
                   'query': 'ABC'
                 }

test/unit/fixture/search_boundary_country.js

@@ -7,7 +7,7 @@ module.exports = {
           'must': [
             {
               'match': {
-                'alpha3': {
+                'parent.country_a': {
                   'analyzer': 'standard',
                   'query': 'ABC'
                 }
@@ -49,15 +49,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -78,15 +74,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_full_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,22 +66,18 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'address.number': {
+              'address_parts.number': {
                 'query': '123',
                 'boost': vs['address:housenumber:boost'],
                 'analyzer': vs['address:housenumber:analyzer']
@@ -93,7 +85,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.street': {
+              'address_parts.street': {
                 'query': 'main st',
                 'boost': vs['address:street:boost'],
                 'analyzer': vs['address:street:analyzer']
@@ -101,7 +93,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.zip': {
+              'address_parts.zip': {
                 'query': '10010',
                 'boost': vs['address:postcode:boost'],
                 'analyzer': vs['address:postcode:analyzer']
@@ -109,55 +101,55 @@ module.exports = {
             }
           }, {
             'match': {
-              'alpha3': {
-                'query': 'USA',
-                'boost': vs['admin:alpha3:boost'],
-                'analyzer': vs['admin:alpha3:analyzer']
+              'parent.country': {
+                'query': 'new york',
+                'boost': vs['admin:country:boost'],
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin0': {
-                'query': 'new york',
-                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+              'parent.country_a': {
+                'query': 'USA',
+                'boost': vs['admin:country_a:boost'],
+                'analyzer': vs['admin:country_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'new york',
-                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'boost': vs['admin:region:boost'],
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'NY',
-                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'boost': vs['admin:region_a:boost'],
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'new york',
-                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'boost': vs['admin:county:boost'],
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'new york',
-                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'boost': vs['admin:localadmin:boost'],
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'new york',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -165,10 +157,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'new york',
-                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'boost': vs['admin:neighbourhood:boost'],
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_linguistic_bbox.js

@@ -39,15 +39,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -68,15 +64,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus_bbox.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_focus_null_island.js

@@ -69,15 +69,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -98,15 +94,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_only.js

@@ -39,15 +39,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -68,15 +64,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]

test/unit/fixture/search_linguistic_viewport.js

@@ -77,18 +77,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             },
             {
@@ -109,18 +105,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
                     'weight': 2
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             }
           ]

test/unit/fixture/search_linguistic_viewport_min_diagonal.js

@@ -77,18 +77,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'popularity'
+                      'field': 'popularity',
+                      'missing': 1
                     },
                     'weight': 1
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'popularity'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             },
             {
@@ -109,18 +105,14 @@ module.exports = {
                   {
                     'field_value_factor': {
                       'modifier': 'log1p',
-                      'field': 'population'
+                      'field': 'population',
+                      'missing': 1
                     },
                     'weight': 2
                   }
                 ],
                 'score_mode': 'first',
-                'boost_mode': 'replace',
-                'filter': {
-                  'exists': {
-                    'field': 'population'
-                  }
-                }
+                'boost_mode': 'replace'
               }
             }
           ]

test/unit/fixture/search_partial_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,62 +66,58 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'admin0': {
+              'parent.country': {
                 'query': 'new york',
-                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+                'boost': vs['admin:country:boost'],
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'new york',
-                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'boost': vs['admin:region:boost'],
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'new york',
-                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'boost': vs['admin:region_a:boost'],
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'new york',
-                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'boost': vs['admin:county:boost'],
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'new york',
-                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'boost': vs['admin:localadmin:boost'],
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'new york',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -133,10 +125,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'new york',
-                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'boost': vs['admin:neighbourhood:boost'],
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_regions_address.js

@@ -41,15 +41,11 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'popularity'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'popularity'
+                  'field': 'popularity',
+                  'missing': 1
                 },
                 'weight': 1
               }]
@@ -70,22 +66,18 @@ module.exports = {
               'max_boost': 20,
               'score_mode': 'first',
               'boost_mode': 'replace',
-              'filter': {
-                'exists': {
-                  'field': 'population'
-                }
-              },
               'functions': [{
                 'field_value_factor': {
                   'modifier': 'log1p',
-                  'field': 'population'
+                  'field': 'population',
+                  'missing': 1
                 },
                 'weight': 2
               }]
             }
           },{
             'match': {
-              'address.number': {
+              'address_parts.number': {
                 'query': '1',
                 'boost': vs['address:housenumber:boost'],
                 'analyzer': vs['address:housenumber:analyzer']
@@ -93,7 +85,7 @@ module.exports = {
             }
           }, {
             'match': {
-              'address.street': {
+              'address_parts.street': {
                 'query': 'water st',
                 'boost': vs['address:street:boost'],
                 'analyzer': vs['address:street:analyzer']
@@ -101,47 +93,47 @@ module.exports = {
             }
           }, {
             'match': {
-              'admin0': {
+              'parent.country': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs['admin:admin0:analyzer']
+                'boost': vs['admin:country:boost'],
+                'analyzer': vs['admin:country:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1': {
+              'parent.region': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs['admin:admin1:analyzer']
+                'boost': vs['admin:region:boost'],
+                'analyzer': vs['admin:region:analyzer']
               }
             }
           }, {
             'match': {
-              'admin1_abbr': {
+              'parent.region_a': {
                 'query': 'NY',
-                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs['admin:admin1_abbr:analyzer']
+                'boost': vs['admin:region_a:boost'],
+                'analyzer': vs['admin:region_a:analyzer']
               }
             }
           }, {
             'match': {
-              'admin2': {
+              'parent.county': {
                 'query': 'manhattan',
-                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs['admin:admin2:analyzer']
+                'boost': vs['admin:county:boost'],
+                'analyzer': vs['admin:county:analyzer']
               }
             }
           }, {
             'match': {
-              'local_admin': {
+              'parent.localadmin': {
                 'query': 'manhattan',
-                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs['admin:local_admin:analyzer']
+                'boost': vs['admin:localadmin:boost'],
+                'analyzer': vs['admin:localadmin:analyzer']
               }
             }
           }, {
             'match': {
-              'locality': {
+              'parent.locality': {
                 'query': 'manhattan',
                 'boost': vs['admin:locality:boost'],
                 'analyzer': vs['admin:locality:analyzer']
@@ -149,10 +141,10 @@ module.exports = {
             }
           }, {
             'match': {
-              'neighborhood': {
+              'parent.neighbourhood': {
                 'query': 'manhattan',
-                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs['admin:neighborhood:analyzer']
+                'boost': vs['admin:neighbourhood:boost'],
+                'analyzer': vs['admin:neighbourhood:analyzer']
               }
             }
           }]

test/unit/helper/adminFields.js

@@ -1,84 +0,0 @@
-var adminFields = require('../../../helper/adminFields');
-
-module.exports.tests = {};
-
-module.exports.tests.interface = function(test, common) {
-  test('validate fields', function(t) {
-    t.assert(adminFields instanceof Function, 'adminFields is a function');
-    t.assert(adminFields() instanceof Array, 'adminFields() returns an array');
-    t.assert(adminFields().length > 0, 'adminFields array is not empty');
-    t.end();
-  });
-};
-
-module.exports.tests.lookupExistance = function(test, common) {
-  test('all expected fields in schema', function(t) {
-
-    var expectedFields = [
-      'one',
-      'two',
-      'three',
-      'four'
-    ];
-    var schema = { mappings: { _default_: { properties: {} } } };
-
-    // inject all expected fields into schema mock
-    expectedFields.forEach(function (field) {
-      schema.mappings._default_.properties[field] = {};
-    });
-
-    var res = adminFields(schema, expectedFields);
-
-    t.deepEquals(res, expectedFields, 'all expected fields are returned');
-    t.end();
-  });
-
-  test('some expected fields in schema', function(t) {
-
-    var expectedFields = [
-      'one',
-      'two',
-      'three',
-      'four'
-    ];
-    var schema = { mappings: { _default_: { properties: {} } } };
-
-    // inject only some of the expected fields into schema mock
-    expectedFields.slice(0, 3).forEach(function (field) {
-      schema.mappings._default_.properties[field] = {};
-    });
-
-    var res = adminFields(schema, expectedFields);
-
-    t.deepEquals(res, expectedFields.slice(0, 3), 'only matching expected fields are returned');
-    t.end();
-  });
-
-  test('no expected fields in schema', function(t) {
-
-    var schema = { mappings: { _default_: { properties: { foo: {} } } } };
-
-    var logErrorCalled = false;
-    var logger = {
-      error: function () {
-        logErrorCalled = true;
-      }};
-
-    var res = adminFields(schema, undefined, logger);
-
-    t.deepEquals(res, [], 'no admin fields found');
-    t.assert(logErrorCalled, 'log error called');
-    t.end();
-  });
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('adminFields: ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/helper/geojsonify.js

@@ -40,7 +40,9 @@ module.exports.tests.search = function(test, common) {
   var input = [
     {
       '_id': 'id1',
-      '_type': 'type1',
+      '_type': 'layer1',
+      'source': 'source1',
+      'layer': 'layer1',
       'center_point': {
         'lat': 51.5337144,
         'lon': -0.1069716
@@ -59,12 +61,6 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'test1',
       'locality': 'test2',
       'neighbourhood': 'test3',
-      'suggest': {
-        'input': [
-          '\'round midnight jazz and blues bar'
-        ],
-        'output': 'osmnode:2208150035'
-      },
       'category': [
         'food',
         'nightlife'
@@ -72,7 +68,9 @@ module.exports.tests.search = function(test, common) {
     },
     {
       '_id': 'id2',
-      '_type': 'type2',
+      '_type': 'layer2',
+      'source': 'source2',
+      'layer': 'layer2',
       'name': {
         'default': 'Blues Cafe'
       },
@@ -88,16 +86,12 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'test1',
       'locality': 'test2',
       'neighbourhood': 'test3',
-      'suggest': {
-        'input': [
-          'blues cafe'
-        ],
-        'output': 'osmway:147495160'
-      }
     },
     {
-      '_id': '34633854',
-      '_type': 'osmway',
+      '_id': 'node:34633854',
+      '_type': 'venue',
+      'source': 'openstreetmap',
+      'layer': 'venue',
       'name': {
         'default': 'Empire State Building'
       },
@@ -113,12 +107,6 @@ module.exports.tests.search = function(test, common) {
       'localadmin': 'Manhattan',
       'locality': 'New York',
       'neighbourhood': 'Koreatown',
-      'suggest': {
-        'input': [
-          'empire state building'
-        ],
-        'output': 'osmway:34633854'
-      },
       'category': [
         'tourism',
         'transport'
@@ -141,9 +129,9 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id1',
-          'gid': 'type1:type1:id1',
-          'layer': 'type1',
-          'source': 'type1',
+          'gid': 'source1:layer1:id1',
+          'layer': 'layer1',
+          'source': 'source1',
           'label': '\'Round Midnight Jazz and Blues Bar, test3, Angel',
           'name': '\'Round Midnight Jazz and Blues Bar',
           'country_a': 'GBR',
@@ -170,9 +158,9 @@ module.exports.tests.search = function(test, common) {
         },
         'properties': {
           'id': 'id2',
-          'gid': 'type2:type2:id2',
-          'layer': 'type2',
-          'source': 'type2',
+          'gid': 'source2:layer2:id2',
+          'layer': 'layer2',
+          'source': 'source2',
           'label': 'Blues Cafe, test3, Smithfield',
           'name': 'Blues Cafe',
           'country_a': 'GBR',
@@ -195,11 +183,11 @@ module.exports.tests.search = function(test, common) {
           ]
         },
         'properties': {
-          'id': '34633854',
-          'gid': 'osm:venue:34633854',
+          'id': 'node:34633854',
+          'gid': 'openstreetmap:venue:node:34633854',
           'layer': 'venue',
-          'source': 'osm',
-          'label': 'Empire State Building, Manhattan, NY',
+          'source': 'openstreetmap',
+          'label': 'Empire State Building, Manhattan, NY, USA',
           'name': 'Empire State Building',
           'country_a': 'USA',
           'country': 'United States',
@@ -219,6 +207,155 @@ module.exports.tests.search = function(test, common) {
     t.deepEqual(json, expected, 'all docs mapped');
     t.end();
   });
+
+  test('filtering out empty items', function (t) {
+    var input = [
+      {
+        'bounding_box': {
+          'min_lat': 40.6514712164,
+          'max_lat': 40.6737320588,
+          'min_lon': -73.8967895508,
+          'max_lon': -73.8665771484
+        },
+        'locality': [
+          'New York'
+        ],
+        'source': 'whosonfirst',
+        'layer': 'neighbourhood',
+        'population': 173198,
+        'popularity': 495,
+        'center_point': {
+          'lon': -73.881319,
+          'lat': 40.663303
+        },
+        'name': {
+          'default': 'East New York'
+        },
+        'source_id': '85816607',
+        'category': [],
+        '_id': '85816607',
+        '_type': 'neighbourhood',
+        '_score': 21.434,
+        'confidence': 0.888,
+        'country': [
+          'United States'
+        ],
+        'country_id': [
+          '85633793'
+        ],
+        'country_a': [
+          'USA'
+        ],
+        'macroregion': [
+          'MacroRegion Name'
+        ],
+        'macroregion_id': [
+          'MacroRegion Id'
+        ],
+        'macroregion_a': [
+          'MacroRegion Abbreviation'
+        ],
+        'region': [
+          'New York'
+        ],
+        'region_id': [
+          '85688543'
+        ],
+        'region_a': [
+          'NY'
+        ],
+        'macrocounty': [
+          'MacroCounty Name'
+        ],
+        'macrocounty_id': [
+          'MacroCounty Id'
+        ],
+        'macrocounty_a': [
+          'MacroCounty Abbreviation'
+        ],
+        'county': [
+          'Kings County'
+        ],
+        'county_id': [
+          '102082361'
+        ],
+        'county_a': [
+          null
+        ],
+        'localadmin': [
+          'Brooklyn'
+        ],
+        'localadmin_id': [
+          '404521211'
+        ],
+        'localadmin_a': [
+          null
+        ],
+        'locality_id': [
+          '85977539'
+        ],
+        'locality_a': [
+          null
+        ],
+        'neighbourhood': [],
+        'neighbourhood_id': []
+      }
+    ];
+
+    var expected = {
+      'type': 'FeatureCollection',
+      'bbox': [-73.8967895508, 40.6514712164, -73.8665771484, 40.6737320588],
+      'features': [
+        {
+          'type': 'Feature',
+          'properties': {
+            'id': '85816607',
+            'gid': 'whosonfirst:neighbourhood:85816607',
+            'layer': 'neighbourhood',
+            'source': 'whosonfirst',
+            'name': 'East New York',
+            'confidence': 0.888,
+            'country': 'United States',
+            'country_id': '85633793',
+            'country_a': 'USA',
+            'macroregion': 'MacroRegion Name',
+            'macroregion_id': 'MacroRegion Id',
+            'macroregion_a': 'MacroRegion Abbreviation',
+            'region': 'New York',
+            'region_id': '85688543',
+            'region_a': 'NY',
+            'macrocounty': 'MacroCounty Name',
+            'macrocounty_id': 'MacroCounty Id',
+            'macrocounty_a': 'MacroCounty Abbreviation',
+            'county': 'Kings County',
+            'county_id': '102082361',
+            'localadmin': 'Brooklyn',
+            'localadmin_id': '404521211',
+            'locality': 'New York',
+            'locality_id': '85977539',
+            'bounding_box': {
+              'min_lat': 40.6514712164,
+              'max_lat': 40.6737320588,
+              'min_lon': -73.8967895508,
+              'max_lon': -73.8665771484
+            },
+            'label': 'East New York, Brooklyn, NY, USA'
+          },
+          'geometry': {
+            'type': 'Point',
+            'coordinates': [
+              -73.881319,
+              40.663303
+            ]
+          }
+        }
+      ]
+    };
+
+    var json = geojsonify.search( input );
+    t.deepEqual(json, expected, 'all wanted properties exposed');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/helper/labelGenerator_GBR.js

@@ -0,0 +1,87 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// GBR street address
+module.exports.tests.one_main_street_uk = function(test, common) {
+  test('one main street uk', function(t) {
+    var doc = {
+      'name': '1 Main St',
+      'housenumber': '1',
+      'street': 'Main St',
+      'postalcode': 'BT77 0BG',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Dungannon'
+    };
+    t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom');
+    t.end();
+  });
+};
+
+// GBR venue
+module.exports.tests.hackney_city_farm = function(test, common) {
+  test('hackney city farm', function(t) {
+    var doc = {
+      'name': 'Hackney City Farm',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Hackney',
+      'county': 'Greater London',
+      'locality': 'London',
+      'neighbourhood': 'Haggerston'
+    };
+    t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London');
+    t.end();
+  });
+};
+
+// GBR country
+module.exports.tests.wales = function(test, common) {
+  test('wales', function(t) {
+    var doc = {
+      'name': 'Wales',
+      'country_a': 'GBR',
+      'country': 'United Kingdom',
+      'region': 'Wales'
+    };
+    t.equal(generator(doc),'Wales, United Kingdom');
+    t.end();
+  });
+};
+
+// GBR macroregion
+module.exports.tests.macroregion_trumps_region = function(test, common) {
+  test('macroregion should trump region when none of neighbourhood, county, localadmin, locality are available', function(t) {
+    var doc = {
+      'name': 'Name',
+      'country_a': 'GBR',
+      'country': 'Country Name',
+      'macroregion': 'Macroregion Name',
+      'region': 'Region Name'
+    };
+
+    t.equal(generator(doc), 'Name, Macroregion Name, Country Name');
+    t.end();
+
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator (GBR): ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelGenerator_SGP.js

@@ -0,0 +1,51 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// SGP region
+module.exports.tests.north_west_singapore = function(test, common) {
+  test('north west singapore', function(t) {
+    var doc = {
+      'name': 'North West',
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'North West'
+    };
+    t.equal(generator(doc),'North West, Singapore');
+    t.end();
+  });
+};
+
+// SGP venue
+module.exports.tests.singapore_mcdonalds = function(test, common) {
+  test('singapore_mcdonalds', function(t) {
+    var doc = {
+      'name': 'McDonald\'s',
+      'country_a': 'SGP',
+      'country': 'Singapore',
+      'region': 'Central Singapore',
+      'locality': 'Singapore'
+    };
+    t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelGenerator_SWE.js

@@ -0,0 +1,53 @@
+
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+// SWE city
+module.exports.tests.skane1 = function(test, common) {
+  test('skåne 1', function(t) {
+    var doc = {
+      'name': 'Malmö',
+      'country_a': 'SWE',
+      'country': 'Sweden',
+      'region': 'Skåne',
+      'county': 'Malmö'
+    };
+    t.equal(generator(doc),'Malmö, Skåne, Sweden');
+    t.end();
+  });
+};
+
+// SWE city
+module.exports.tests.skane2 = function(test, common) {
+  test('skåne 2', function(t) {
+    var doc = {
+      'name': 'Malmö',
+      'country_a': 'SWE',
+      'country': 'Sweden',
+      'region': 'Skåne',
+      'county': 'Malmö',
+      'locality': 'Malmö'
+    };
+    t.equal(generator(doc),'Malmö, Skåne, Sweden');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelGenerator_USA.js

@@ -0,0 +1,229 @@
+var generator = require('../../../helper/labelGenerator');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof generator, 'function', 'valid function');
+    t.end();
+  });
+};
+
+module.exports.tests.localadmin = function(test, common) {
+  test('localadmin should trump locality, neighbourhood, and county', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'county': 'County Name',
+      'localadmin': 'LocalAdmin Name',
+      'locality': 'Locality Name',
+      'neighbourhood': 'Neighbourhood Name'
+    };
+    t.equal(generator(doc),'Default Name, LocalAdmin Name, Region Abbr, USA');
+    t.end();
+  });
+};
+
+module.exports.tests.locality = function(test, common) {
+  test('locality should trump neighbourhood and county when localadmin not available', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'county': 'County Name',
+      'locality': 'Locality Name',
+      'neighbourhood': 'Neighbourhood Name'
+    };
+    t.equal(generator(doc),'Default Name, Locality Name, Region Abbr, USA');
+    t.end();
+  });
+};
+
+module.exports.tests.neighbourhood = function(test, common) {
+  test('neighbourhood should trump county when neither localadmin nor locality', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'county': 'County Name',
+      'neighbourhood': 'Neighbourhood Name'
+    };
+    t.equal(generator(doc),'Default Name, Neighbourhood Name, Region Abbr, USA');
+    t.end();
+  });
+};
+
+module.exports.tests.county = function(test, common) {
+  test('county should be used when localadmin, locality, and neighbourhood are not available', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'county': 'County Name'
+    };
+    t.equal(generator(doc),'Default Name, County Name, Region Abbr, USA');
+    t.end();
+  });
+};
+
+module.exports.tests.region = function(test, common) {
+  test('region should be used when region_a is not available', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name'
+    };
+    t.equal(generator(doc),'Default Name, Region Name, USA');
+    t.end();
+  });
+};
+
+// USA geonames state
+module.exports.tests.region_geonames = function(test, common) {
+  test('default name should not be prepended when source=geonames and layer=region', function(t) {
+    var doc = {
+      'name': 'Region Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'source': 'geonames',
+      'layer': 'region'
+    };
+    t.equal(generator(doc),'Region Name, USA');
+    t.end();
+  });
+};
+
+// USA whosonfirst state
+module.exports.tests.region_whosonfirst = function(test, common) {
+  test('default name should not be prepended when source=whosonfirst and layer=region', function(t) {
+    var doc = {
+      'name': 'Region Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'source': 'whosonfirst',
+      'layer': 'region'
+    };
+    t.equal(generator(doc),'Region Name, USA');
+    t.end();
+  });
+};
+
+// USA non-geonames/whosonfirst state
+module.exports.tests.region_other_source = function(test, common) {
+  test('default name should be prepended when layer=region and source is not whosonfirst or geonames', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'Region Name',
+      'region_a': 'Region Abbr',
+      'source': 'not geonames or whosonfirst',
+      'layer': 'region'
+    };
+    t.equal(generator(doc),'Default Name, Region Name, USA',generator(doc));
+    t.end();
+  });
+};
+
+// major USA city
+module.exports.tests.san_francisco = function(test, common) {
+  test('san francisco', function(t) {
+    var doc = {
+      'name': 'San Francisco',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'California',
+      'region_a': 'CA',
+      'county': 'San Francisco County',
+      'locality': 'San Francisco'
+    };
+    t.equal(generator(doc),'San Francisco, San Francisco County, CA, USA');
+    t.end();
+  });
+};
+
+// USA venue
+module.exports.tests.nyc_office = function(test, common) {
+  test('30 West 26th Street', function(t) {
+    var doc = {
+      'name': '30 West 26th Street',
+      'housenumber': '30',
+      'street': 'West 26th Street',
+      'postalcode': '10010',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'New York',
+      'region_a': 'NY',
+      'county': 'New York County',
+      'localadmin': 'Manhattan',
+      'locality': 'New York',
+      'neighbourhood': 'Flatiron District'
+    };
+    t.equal(generator(doc),'30 West 26th Street, Manhattan, NY, USA');
+    t.end();
+  });
+};
+
+// USA NYC eatery
+module.exports.tests.nyc_bakery = function(test, common) {
+  test('New York Bakery', function(t) {
+    var doc = {
+      'name': 'New York Bakery',
+      'housenumber': '51 W',
+      'street': '29th',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'New York',
+      'region_a': 'NY',
+      'county': 'New York County',
+      'localadmin': 'Manhattan',
+      'locality': 'New York',
+      'neighbourhood': 'Koreatown'
+    };
+    t.equal(generator(doc),'New York Bakery, Manhattan, NY, USA');
+    t.end();
+  });
+};
+
+// USA SFC building
+module.exports.tests.ferry_building = function(test, common) {
+  test('Ferry Building', function(t) {
+    var doc = {
+      'name': 'Ferry Building',
+      'country_a': 'USA',
+      'country': 'United States',
+      'region': 'California',
+      'region_a': 'CA',
+      'county': 'San Francisco County',
+      'locality': 'San Francisco',
+      'neighbourhood': 'Financial District'
+    };
+    t.equal(generator(doc),'Ferry Building, San Francisco, CA, USA');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('label generator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/labelGenerator_default.js

@@ -10,89 +10,11 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-// major USA city
-module.exports.tests.san_francisco = function(test, common) {
-  test('san francisco', function(t) {
-    var doc = {
-      'name': { 'default': 'San Francisco' },
-      'country_a': 'USA',
-      'country': 'United States',
-      'region': 'California',
-      'region_a': 'CA',
-      'county': 'San Francisco County',
-      'locality': 'San Francisco'
-    };
-    t.equal(generator(doc),'San Francisco, San Francisco County, CA');
-    t.end();
-  });
-};
-
-// USA venue
-module.exports.tests.nyc_office = function(test, common) {
-  test('30 West 26th Street', function(t) {
-    var doc = {
-      'name': { 'default': '30 West 26th Street' },
-      'housenumber': '30',
-      'street': 'West 26th Street',
-      'postalcode': '10010',
-      'country_a': 'USA',
-      'country': 'United States',
-      'region': 'New York',
-      'region_a': 'NY',
-      'county': 'New York County',
-      'localadmin': 'Manhattan',
-      'locality': 'New York',
-      'neighbourhood': 'Flatiron District'
-    };
-    t.equal(generator(doc),'30 West 26th Street, Manhattan, NY');
-    t.end();
-  });
-};
-
-// USA NYC eatery
-module.exports.tests.nyc_bakery = function(test, common) {
-  test('New York Bakery', function(t) {
-    var doc = {
-      'name': { 'default': 'New York Bakery' },
-      'housenumber': '51 W',
-      'street': '29th',
-      'country_a': 'USA',
-      'country': 'United States',
-      'region': 'New York',
-      'region_a': 'NY',
-      'county': 'New York County',
-      'localadmin': 'Manhattan',
-      'locality': 'New York',
-      'neighbourhood': 'Koreatown'
-    };
-    t.equal(generator(doc),'New York Bakery, Manhattan, NY');
-    t.end();
-  });
-};
-
-// USA SFC building
-module.exports.tests.ferry_building = function(test, common) {
-  test('Ferry Building', function(t) {
-    var doc = {
-      'name': { 'default': 'Ferry Building' },
-      'country_a': 'USA',
-      'country': 'United States',
-      'region': 'California',
-      'region_a': 'CA',
-      'county': 'San Francisco County',
-      'locality': 'San Francisco',
-      'neighbourhood': 'Financial District'
-    };
-    t.equal(generator(doc),'Ferry Building, San Francisco, CA');
-    t.end();
-  });
-};
-
 // AUS state
 module.exports.tests.new_south_wales = function(test, common) {
   test('new south wales', function(t) {
     var doc = {
-      'name': { 'default': 'New South Wales' },
+      'name': 'New South Wales',
       'country_a': 'AUS',
       'country': 'Australia',
       'region': 'New South Wales'
@@ -102,26 +24,11 @@ module.exports.tests.new_south_wales = function(test, common) {
   });
 };
 
-// USA state
-module.exports.tests.california = function(test, common) {
-  test('california', function(t) {
-    var doc = {
-      'name': { 'default': 'California' },
-      'country_a': 'USA',
-      'country': 'United States',
-      'region': 'California',
-      'region_a': 'CA'
-    };
-    t.equal(generator(doc),'California, CA');
-    t.end();
-  });
-};
-
 // IND state
 module.exports.tests.west_bengal = function(test, common) {
   test('west bengal', function(t) {
     var doc = {
-      'name': { 'default': 'West Bengal' },
+      'name': 'West Bengal',
       'country_a': 'IND',
       'country': 'India',
       'region': 'West Bengal'
@@ -135,7 +42,7 @@ module.exports.tests.west_bengal = function(test, common) {
 module.exports.tests.bangalore = function(test, common) {
   test('bangalore', function(t) {
     var doc = {
-      'name': { 'default': 'Bangalore' },
+      'name': 'Bangalore',
       'country_a': 'IND',
       'country': 'India',
       'region': 'Karnataka',
@@ -151,7 +58,7 @@ module.exports.tests.bangalore = function(test, common) {
 module.exports.tests.sarjapur = function(test, common) {
   test('Sarjapur', function(t) {
     var doc = {
-      'name': { 'default': 'Sarjapur' },
+      'name': 'Sarjapur',
       'country_a': 'IND',
       'country': 'India',
       'region': 'Karnataka'
@@ -165,7 +72,7 @@ module.exports.tests.sarjapur = function(test, common) {
 module.exports.tests.bengaluru_east = function(test, common) {
   test('Bengaluru East', function(t) {
     var doc = {
-      'name': { 'default': 'Bengaluru East' },
+      'name': 'Bengaluru East',
       'country_a': 'IND',
       'country': 'India',
       'region': 'Karnataka',
@@ -183,7 +90,7 @@ module.exports.tests.bengaluru_east = function(test, common) {
 module.exports.tests.wellington_victoria = function(test, common) {
   test('Wellington, Victoria, Australia', function(t) {
     var doc = {
-      'name': { 'default': 'Wellington' },
+      'name': 'Wellington',
       'country_a': 'AUS',
       'country': 'Australia',
       'region': 'Victoria',
@@ -194,25 +101,11 @@ module.exports.tests.wellington_victoria = function(test, common) {
   });
 };
 
-// SGP region
-module.exports.tests.north_west_singapore = function(test, common) {
-  test('north west singapore', function(t) {
-    var doc = {
-      'name': { 'default': 'North West' },
-      'country_a': 'SGP',
-      'country': 'Singapore',
-      'region': 'North West'
-    };
-    t.equal(generator(doc),'North West, Singapore');
-    t.end();
-  });
-};
-
 // IRQ region
 module.exports.tests.arbil = function(test, common) {
   test('arbil', function(t) {
     var doc = {
-      'name': { 'default': 'Arbil' },
+      'name': 'Arbil',
       'country_a': 'IRQ',
       'country': 'Iraq',
       'region': 'Arbil'
@@ -226,7 +119,7 @@ module.exports.tests.arbil = function(test, common) {
 module.exports.tests.madrid = function(test, common) {
   test('madrid', function(t) {
     var doc = {
-      'name': { 'default': 'Madrid' },
+      'name': 'Madrid',
       'country_a': 'ESP',
       'country': 'Spain',
       'region': 'Madrid'
@@ -236,76 +129,11 @@ module.exports.tests.madrid = function(test, common) {
   });
 };
 
-// SWE city
-module.exports.tests.skane1 = function(test, common) {
-  test('skåne 1', function(t) {
-    var doc = {
-      'name': { 'default': 'Malmö' },
-      'country_a': 'SWE',
-      'country': 'Sweden',
-      'region': 'Skåne',
-      'county': 'Malmö'
-    };
-    t.equal(generator(doc),'Malmö, Skåne, Sweden');
-    t.end();
-  });
-};
-
-// SWE city
-module.exports.tests.skane2 = function(test, common) {
-  test('skåne 2', function(t) {
-    var doc = {
-      'name': { 'default': 'Malmö' },
-      'country_a': 'SWE',
-      'country': 'Sweden',
-      'region': 'Skåne',
-      'county': 'Malmö',
-      'locality': 'Malmö'
-    };
-    t.equal(generator(doc),'Malmö, Skåne, Sweden');
-    t.end();
-  });
-};
-
-// GBR street address
-module.exports.tests.one_main_street_uk = function(test, common) {
-  test('one main street uk', function(t) {
-    var doc = {
-      'name': { 'default': '1 Main St' },
-      'housenumber': '1',
-      'street': 'Main St',
-      'postalcode': 'BT77 0BG',
-      'country_a': 'GBR',
-      'country': 'United Kingdom',
-      'region': 'Dungannon'
-    };
-    t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom');
-    t.end();
-  });
-};
-
-// GBR venue
-module.exports.tests.hackney_city_farm = function(test, common) {
-  test('hackney city farm', function(t) {
-    var doc = {
-      'name': { 'default': 'Hackney City Farm' },
-      'country_a': 'GBR',
-      'country': 'United Kingdom',
-      'region': 'Hackney',
-      'county': 'Greater London',
-      'locality': 'London',
-      'neighbourhood': 'Haggerston'
-    };
-    t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London');
-    t.end();
-  });
-};
-
 // DEU street address
 module.exports.tests.one_grolmanstrasse = function(test, common) {
   test('one grolmanstrasse', function(t) {
     var doc = {
-      'name': { 'default': '1 Grolmanstraße' },
+      'name': '1 Grolmanstraße',
       'housenumber': '1',
       'street': 'Grolmanstraße',
       'postalcode': '10623',
@@ -325,7 +153,7 @@ module.exports.tests.one_grolmanstrasse = function(test, common) {
 module.exports.tests.new_zealand = function(test, common) {
   test('new zealand', function(t) {
     var doc = {
-      'name': { 'default': 'New Zealand' },
+      'name': 'New Zealand',
       'country_a': 'NZL',
       'country': 'New Zealand'
     };
@@ -334,25 +162,11 @@ module.exports.tests.new_zealand = function(test, common) {
   });
 };
 
-// GBR country
-module.exports.tests.wales = function(test, common) {
-  test('wales', function(t) {
-    var doc = {
-      'name': { 'default': 'Wales' },
-      'country_a': 'GBR',
-      'country': 'United Kingdom',
-      'region': 'Wales'
-    };
-    t.equal(generator(doc),'Wales, United Kingdom');
-    t.end();
-  });
-};
-
 // IRL country
 module.exports.tests.republic_of_ireland = function(test, common) {
   test('northern ireland', function(t) {
     var doc = {
-      'name': { 'default': 'Ireland' },
+      'name': 'Ireland',
       'country_a': 'IRL',
       'country': 'Ireland'
     };
@@ -362,26 +176,11 @@ module.exports.tests.republic_of_ireland = function(test, common) {
   });
 };
 
-// SGP venue
-module.exports.tests.singapore_mcdonalds = function(test, common) {
-  test('singapore_mcdonalds', function(t) {
-    var doc = {
-      'name': { 'default': 'McDonald\'s' },
-      'country_a': 'SGP',
-      'country': 'Singapore',
-      'region': 'Central Singapore',
-      'locality': 'Singapore'
-    };
-    t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore');
-    t.end();
-  });
-};
-
 // THA province
 module.exports.tests.krabi_province = function(test, common) {
   test('Krabi Provence', function(t) {
     var doc = {
-      'name': { 'default': 'Krabi' },
+      'name': 'Krabi',
       'country_a': 'THA',
       'country': 'Thailand',
       'region': 'Krabi'
@@ -395,7 +194,7 @@ module.exports.tests.krabi_province = function(test, common) {
 module.exports.tests.koh_lanta = function(test, common) {
   test('Koh Lanta', function(t) {
     var doc = {
-      'name': { 'default': 'Ko Lanta' },
+      'name': 'Ko Lanta',
       'country_a': 'THA',
       'country': 'Thailand',
       'region': 'Krabi'
@@ -409,7 +208,7 @@ module.exports.tests.koh_lanta = function(test, common) {
 module.exports.tests.black_dog_cafe = function(test, common) {
   test('Black Dog Cafe', function(t) {
     var doc = {
-      'name': { 'default': 'Black Dog Cafe' },
+      'name': 'Black Dog Cafe',
       'country_a': 'NZL',
       'country': 'New Zealand',
       'region': 'Auckland Region',
@@ -424,7 +223,7 @@ module.exports.tests.black_dog_cafe = function(test, common) {
 module.exports.tests.beach_bablyon = function(test, common) {
   test('Beach Bablyon', function(t) {
     var doc = {
-      'name': { 'default': 'Beach Bablyon' },
+      'name': 'Beach Bablyon',
       'country_a': 'NZL',
       'country': 'New Zealand',
       'region': 'Wellington Region',
@@ -441,7 +240,7 @@ module.exports.tests.beach_bablyon = function(test, common) {
 module.exports.tests.waiotapu = function(test, common) {
   test('Waiotapu', function(t) {
     var doc = {
-      'name': { 'default': 'Waiotapu' },
+      'name': 'Waiotapu',
       'country_a': 'NZL',
       'country': 'New Zealand',
       'region': 'Bay of Plenty',
@@ -452,6 +251,57 @@ module.exports.tests.waiotapu = function(test, common) {
   });
 };
 
+
+module.exports.tests.non_us_or_ca_region = function(test, common) {
+  test('geonames US', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'XYZ',
+      'country': 'Full Country Name',
+      'region': 'Full Region Name',
+      'layer': 'region',
+      'source': 'geonames'
+    };
+
+    t.equal(generator(doc), 'Default Name, Full Region Name, Full Country Name');
+    t.end();
+
+  });
+
+  test('whosonfirst US', function(t) {
+    var doc = {
+      'name': 'Default Name',
+      'country_a': 'XYZ',
+      'country': 'Full Country Name',
+      'region': 'Full Region Name',
+      'layer': 'region',
+      'source': 'whosonfirst'
+    };
+
+    t.equal(generator(doc), 'Default Name, Full Region Name, Full Country Name');
+    t.end();
+
+  });
+
+};
+
+// macroregion
+module.exports.tests.macroregion_trumps_region = function(test, common) {
+  test('macroregion should trump region when none of localadmin, locality, neighbourhood, county are available', function(t) {
+    var doc = {
+      'name': 'Name',
+      'country_a': 'Country abbreviation',
+      'country': 'Country Name',
+      'macroregion': 'Macroregion Name',
+      'region': 'Region Name'
+    };
+
+    t.equal(generator(doc), 'Name, Macroregion Name, Country Name');
+    t.end();
+
+  });
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {

test/unit/helper/labelSchema.js

@@ -1,5 +1,5 @@
 
-var schemas = require('../../../helper/labelSchema.json');
+var schemas = require('../../../helper/labelSchema');
 var alpha3  = require('../mock/alpha3.json');
 
 module.exports.tests = {};
@@ -12,36 +12,766 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-module.exports.tests.valid = function(test, common) {
-  var valid_keys = ['localadmin', 'locality', 'neighbourhood', 'county', 'region_a', 'region', 'country'];
-  var default_schema = {
-    local: ['localadmin', 'locality', 'neighbourhood', 'county', 'region'],
-    regional: ['country']
-  };
-
-  var isValid = function(keys, schema) {
-    test('valid key/object (' + keys + ')' , function(t) {
-      if (keys === 'default') {
-        t.deepEqual(schema, default_schema, 'valid default schema');
-      } else {
-        t.equal(alpha3.hasOwnProperty(keys), true, 'valid key');
-      }
-      t.equal(typeof schema, 'object', 'valid object');
-      t.notEqual(Object.getOwnPropertyNames(schema).length, 0,  'object not empty');
-      for (var levels in schema) {
-        t.equal(Object.prototype.toString.call(schema[levels]), '[object Array]', levels+' is an array');
-        for (var i=0;i<schema[levels].length;i++) {
-          var key = schema[levels][i];
-          t.notEqual(valid_keys.indexOf(key), -1, key + ' is valid');
-        }
-      }
-      t.end();
-    });
-  };
-
-  for (var keys in schemas) {
-    isValid(keys, schemas[keys]);
-  }
+module.exports.tests.supported_countries = function(test, common) {
+  test('support countries', function(t) {
+    var supported_countries = Object.keys(schemas);
+
+    t.notEquals(supported_countries.indexOf('USA'), -1);
+    t.notEquals(supported_countries.indexOf('GBR'), -1);
+    t.notEquals(supported_countries.indexOf('SGP'), -1);
+    t.notEquals(supported_countries.indexOf('SWE'), -1);
+    t.notEquals(supported_countries.indexOf('default'), -1);
+    t.equals(supported_countries.length, 5);
+
+    t.equals(Object.keys(schemas.USA).length, 3);
+    t.equals(Object.keys(schemas.GBR).length, 2);
+    t.equals(Object.keys(schemas.SGP).length, 2);
+    t.equals(Object.keys(schemas.SWE).length, 2);
+    t.equals(Object.keys(schemas.default).length, 2);
+
+    t.end();
+
+  });
+};
+
+module.exports.tests.usa = function(test, common) {
+  test('USA.local should use localadmin value over locality, neighbourhood, and county', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      neighbourhood: 'neighbourhood value',
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']);
+    t.end();
+
+  });
+
+  test('USA.local should use locality value over neighbourhood and county when no localadmin', function(t) {
+    var record = {
+      locality: 'locality value',
+      neighbourhood: 'neighbourhood value',
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'locality value']);
+    t.end();
+
+  });
+
+  test('USA.local should use neighbourhood value over county when no localadmin or locality', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']);
+    t.end();
+
+  });
+
+  test('USA.local should use county value when no localadmin, locality, or neighbourhood', function(t) {
+    var record = {
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('USA.local should not modify labelParts if none of localadmin, locality, neighbourhood, or county is available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+  test('USA.regional should use region when layer=region and region is available', function(t) {
+    var record = {
+      layer: 'region',
+      region: 'region name',
+      region_a: 'region_a name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region name']);
+    t.end();
+
+  });
+
+  test('USA.regional should use region_a when layer=region and region is unavailable', function(t) {
+    var record = {
+      layer: 'region',
+      region_a: 'region_a name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region_a name']);
+    t.end();
+
+  });
+
+  test('USA.regional should use region_a when layer!=region and both region and region_a are available', function(t) {
+    var record = {
+      layer: 'not region',
+      region: 'region name',
+      region_a: 'region_a name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region_a name']);
+    t.end();
+
+  });
+
+  test('USA.regional should use region when layer!=region and region_a is unavailable', function(t) {
+    var record = {
+      layer: 'region',
+      region: 'region name',
+      region_a: 'region_a name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region name'], 'region should have been appended');
+    t.end();
+
+  });
+
+  test('USA.regional should not append anything when neither region nor region_a are available', function(t) {
+    var record = {
+      layer: 'region',
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value'], 'no USA.region should have appended');
+    t.end();
+
+  });
+
+  test('USA.country should append country_a when available', function(t) {
+    var record = {
+      country_a: 'country_a name',
+      country: 'country name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.country;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'country_a name'], 'country_a should have appended');
+    t.end();
+
+  });
+
+  test('USA.country should not append anything when country_a is unavailable', function(t) {
+    var record = {
+      country: 'country name'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.USA.country;
+
+    t.deepEqual(f(record, labelParts), ['initial value'], 'no USA.country should have appended');
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.gbr = function(test, common) {
+  test('GBR.local should use neighbourhood value over county, localadmin, locality, region', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']);
+    t.end();
+
+  });
+
+  test('GBR.local should use county value over county, localadmin, locality, region', function(t) {
+    var record = {
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('GBR.local should use localadmin value over locality, region', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']);
+    t.end();
+
+  });
+
+  test('GBR.local should use locality value over region', function(t) {
+    var record = {
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'locality value']);
+    t.end();
+
+  });
+
+  test('GBR.local should use region value when nothing else is available', function(t) {
+    var record = {
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('GBR.local should not append anything when none of neighbourhood, county, localadmin, locality, region are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+  test('GBR.regional should use county over country and region', function(t) {
+    var record = {
+      county: 'county value',
+      country: 'country value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('GBR.regional should use country over region', function(t) {
+    var record = {
+      country: 'country value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'country value']);
+    t.end();
+
+  });
+
+  test('GBR.regional should use region when county and country aren not available', function(t) {
+    var record = {
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('GBR.regional should not append anything when none of county, country, or region are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.GBR.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.sgp = function(test, common) {
+  test('SGP.local should use neighbourhood value over region, county, localadmin, locality', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']);
+    t.end();
+
+  });
+
+  test('SGP.local should use region value over county, localadmin, locality', function(t) {
+    var record = {
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('SGP.local should use county value over localadmin, locality', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('SGP.local should use localadmin value over locality', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']);
+    t.end();
+
+  });
+
+  test('SGP.local should use locality value when nothing else is available', function(t) {
+    var record = {
+      locality: 'locality value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'locality value']);
+    t.end();
+
+  });
+
+  test('SGP.local should not append anything when none of neighbourhood, region, county, localadmin, locality are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should use county over country and region', function(t) {
+    var record = {
+      county: 'county value',
+      country: 'country value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should use country over region', function(t) {
+    var record = {
+      country: 'country value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'country value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should use region when county and country aren not available', function(t) {
+    var record = {
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should not append anything when none of county, country, or region are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.swe = function(test, common) {
+  test('SWE.local should use neighbourhood value over region, county, localadmin, locality', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']);
+    t.end();
+
+  });
+
+  test('SWE.local should use region value over county, localadmin, locality', function(t) {
+    var record = {
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('SWE.local should use county value over localadmin, locality', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      county: 'county value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('SWE.local should use localadmin value over locality', function(t) {
+    var record = {
+      localadmin: 'localadmin value',
+      locality: 'locality value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']);
+    t.end();
+
+  });
+
+  test('SWE.local should use locality value when nothing else is available', function(t) {
+    var record = {
+      locality: 'locality value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'locality value']);
+    t.end();
+
+  });
+
+  test('SWE.local should not append anything when none of neighbourhood, region, county, localadmin, locality are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SWE.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should use country when available', function(t) {
+    var record = {
+      country: 'country value',
+      country_a: 'country_a value',
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'country value']);
+    t.end();
+
+  });
+
+  test('SGP.regional should not append anything when country is not available', function(t) {
+    var record = {
+      country_a: 'country_a value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.SGP.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+};
+
+module.exports.tests.default = function(test, common) {
+  test('default.local should use localadmin value over locality, neighbourhood, county, region', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      localadmin: 'localadmin value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']);
+    t.end();
+
+  });
+
+  test('default.local should use locality value over neighbourhood, county, region', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      locality: 'locality value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'locality value']);
+    t.end();
+
+  });
+
+  test('default.local should use neighbourhood value over county, region', function(t) {
+    var record = {
+      neighbourhood: 'neighbourhood value',
+      county: 'county value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']);
+    t.end();
+
+  });
+
+  test('default.local should use county value over region', function(t) {
+    var record = {
+      county: 'county value',
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'county value']);
+    t.end();
+
+  });
+
+  test('default.local should use region value when nothing else is available', function(t) {
+    var record = {
+      region: 'region value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'region value']);
+    t.end();
+
+  });
+
+  test('default.local should not append anything when none of neighbourhood, region, county, localadmin, ' +
+        'locality are available', function(t) {
+    var record = {};
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.local;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
+  test('default.regional should use country over region, region_a, or country_a', function(t) {
+    var record = {
+      region: 'region value',
+      region_a: 'region_a value',
+      country: 'country value',
+      country_a: 'country_a value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value', 'country value']);
+    t.end();
+
+  });
+
+  test('default.regional should not append any value if country is not available', function(t) {
+    var record = {
+      region: 'region value',
+      region_a: 'region_a value',
+      country_a: 'country_a value'
+    };
+
+    var labelParts = ['initial value'];
+
+    var f = schemas.default.regional;
+
+    t.deepEqual(f(record, labelParts), ['initial value']);
+    t.end();
+
+  });
+
 };
 
 module.exports.all = function (tape, common) {

test/unit/helper/text_parser.js

@@ -1,7 +1,7 @@
 var parser = require('../../../helper/text_parser');
 
 var type_mapping = require('../../../helper/type_mapping');
-var layers_map = type_mapping.layer_with_aliases_to_type;
+var layers_map = type_mapping.layer_mapping;
 
 module.exports.tests = {};
 

test/unit/helper/type_mapping.js

@@ -4,132 +4,31 @@ var type_mapping = require('../../../helper/type_mapping');
 module.exports.tests = {};
 
 module.exports.tests.interfaces = function(test, common) {
-  test('types list', function(t) {
-    t.ok(check.array(type_mapping.types), 'is array');
-    t.ok(check.hasLength(type_mapping.types, 11), 'has correct number of elements');
+  test('basic layer mapping', function(t) {
+    t.deepEquals(type_mapping.layer_mapping.venue, ['venue']);
+    t.deepEquals(type_mapping.layer_mapping.address, ['address']);
     t.end();
   });
 
-  test('type to source mapping', function(t) {
-    t.ok(check.object(type_mapping.type_to_source), 'is object');
-    t.ok(check.hasLength(Object.keys(type_mapping.type_to_source), 11), 'has correct number of elements');
+  test('alias layer mapping', function(t) {
+    t.deepEquals(type_mapping.layer_mapping.coarse,
+                 [ 'continent', 'macrocountry', 'country', 'dependency',
+                   'region', 'locality', 'localadmin', 'county', 'macrohood',
+                   'neighbourhood', 'microhood', 'disputed' ]);
     t.end();
   });
 
-  test('type to layer mapping', function(t) {
-    t.ok(check.object(type_mapping.type_to_layer), 'is object');
-    t.ok(check.hasLength(Object.keys(type_mapping.type_to_layer), 11), 'has correct number of elements');
+  test('basic source mapping', function(t) {
+    t.deepEquals(type_mapping.source_mapping.osm, ['openstreetmap']);
+    t.deepEquals(type_mapping.source_mapping.openaddresses, ['openaddresses']);
     t.end();
   });
 
-  test('source to type mapping', function(t) {
-    t.ok(check.object(type_mapping.source_to_type), 'is object');
-    t.ok(check.hasLength(Object.keys(type_mapping.source_to_type), 8), 'has correct number of elements');
+  test('alias source mapping', function(t) {
+    t.deepEquals(type_mapping.source_mapping.openstreetmap, ['openstreetmap']);
+    t.deepEquals(type_mapping.source_mapping.wof, ['whosonfirst']);
     t.end();
   });
-
-  test('layer to type mapping', function(t) {
-    t.ok(check.object(type_mapping.layer_to_type), 'is object');
-    t.equal(Object.keys(type_mapping.layer_to_type).length, 8, 'has correct number of elements');
-    t.end();
-  });
-
-  test('layer to type mapping (with aliases)', function(t) {
-    t.ok(check.object(type_mapping.layer_with_aliases_to_type), 'is object');
-    t.ok(check.hasLength(Object.keys(type_mapping.layer_with_aliases_to_type), 9), 'has correct number of elements');
-    t.end();
-  });
-
-  test('\'osm\' and \'openstreetmap\' sources should only successfully map to \'venue\' and \'address\' layers', function(t) {
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'venue'), ['osmnode', 'osmway']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'address'), ['osmaddress']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'coarse'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'venue'), ['osmnode', 'osmway']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'address'), ['osmaddress']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'coarse'), []);
-    t.end();
-  });
-
-  test('\'gn\' and \'geonames\' sources should only successfully map to \'venue\' layers', function(t) {
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'venue'), ['geoname']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'address'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'coarse'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'venue'), ['geoname']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'address'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'coarse'), []);
-    t.end();
-  });
-
-  test('\'oa\' and \'openaddresses\' sources should only successfully map to \'address\' layer', function(t) {
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'venue'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'address'), ['openaddresses']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'coarse'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'venue'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'address'), ['openaddresses']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'country'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'region'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'county'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'locality'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'localadmin'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'neighbourhood'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'coarse'), []);
-    t.end();
-  });
-
-  test('\'qs\' and \'quattroshapes\' sources should only successfully map to \'address\' layer', function(t) {
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'venue'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'address'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'country'), ['admin0']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'region'), ['admin1']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'county'), ['admin2']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'locality'), ['locality']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'localadmin'), ['local_admin']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'neighbourhood'), ['neighborhood']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'coarse'),
-      ['admin0','admin1','admin2','neighborhood','locality','local_admin']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'venue'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'address'), []);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'country'), ['admin0']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'region'), ['admin1']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'county'), ['admin2']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'locality'), ['locality']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'localadmin'), ['local_admin']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'neighbourhood'), ['neighborhood']);
-    t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'coarse'),
-      ['admin0','admin1','admin2','neighborhood','locality','local_admin']);
-    t.end();
-  });
-
 };
 
 module.exports.all = function (tape, common) {

test/unit/helper/types.js

@@ -1,92 +0,0 @@
-var types = require('../../../helper/types');
-
-module.exports.tests = {};
-
-module.exports.tests.no_cleaned_types = function(test, common) {
-  test('no cleaned types', function(t) {
-    function testIt() {
-      types({});
-    }
-
-    t.throws(testIt, /clean_types should not be null or undefined/, 'no input should result in exception');
-
-    t.end();
-  });
-};
-
-module.exports.tests.address_parser = function(test, common) {
-  test('address parser specifies only admin layers', function(t) {
-    var cleaned_types = {
-      from_text_parser: ['admin0'] // simplified return value from address parser
-    };
-    var actual = types(cleaned_types);
-    var expected = ['admin0']; // simplified expected value for all admin layers
-    t.deepEqual(actual, expected, 'only layers specified by address parser returned');
-    t.end();
-  });
-};
-
-module.exports.tests.layers_parameter = function(test, common) {
-  test('layers parameter specifies only some layers', function(t) {
-    var cleaned_types = {
-      from_layers: ['geoname']
-    };
-    var actual = types(cleaned_types);
-    var expected = ['geoname'];
-    t.deepEqual(actual, expected, 'only types specified by layers parameter returned');
-    t.end();
-  });
-};
-
-module.exports.tests.layers_parameter_and_address_parser = function(test, common) {
-  test('layers parameter and address parser present', function(t) {
-    var cleaned_types = {
-      from_layers: ['geoname'],
-      from_text_parser: ['admin0'] // simplified return value from address parse
-    };
-    var actual = types(cleaned_types);
-    var expected = ['geoname'];
-    t.deepEqual(actual, expected, 'layers parameter overrides address parser completely');
-    t.end();
-  });
-};
-
-module.exports.tests.source_parameter = function(test, common) {
-  test('source parameter specified', function(t) {
-    var cleaned_types = {
-      from_sources: ['openaddresses']
-    };
-
-    var actual = types(cleaned_types);
-
-    var expected = ['openaddresses'];
-    t.deepEqual(actual, expected, 'type parameter set to types specified by source');
-    t.end();
-  });
-};
-
-module.exports.tests.source_and_layers_parameters = function(test, common) {
-  test('source and layers parameter both specified', function(t) {
-    var cleaned_types = {
-      from_sources: ['openaddresses'],
-      from_layers: ['osmaddress', 'openaddresses']
-    };
-
-    var actual = types(cleaned_types);
-
-    var expected = ['openaddresses'];
-    t.deepEqual(actual, expected, 'type set to intersection of source and layer types');
-    t.end();
-  });
-};
-
-module.exports.all = function (tape, common) {
-
-  function test(name, testFunction) {
-    return tape('types: ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

test/unit/middleware/confidenceScore.js

@@ -70,13 +70,21 @@ module.exports.tests.confidenceScore = function(test, common) {
         value: 1,
         center_point: { lat: 100.1, lon: -50.5 },
         name: { default: 'test name1' },
-        admin0: 'country1', admin1: 'state1', admin2: 'city1'
+        parent: {
+          country: ['country1'],
+          region: ['state1'],
+          county: ['city1']
+        }
       }, {
+        _score: 20,
         value: 2,
         center_point: { lat: 100.2, lon: -51.5 },
         name: { default: 'test name2' },
-        admin0: 'country2', admin1: 'state2', admin2: 'city2',
-        _score: 20
+        parent: {
+          country: ['country2'],
+          region: ['state2'],
+          county: ['city2']
+        }
       }],
       meta: {scores: [10]}
     };

test/unit/middleware/dedupe.js

@@ -16,7 +16,7 @@ module.exports.tests.dedupe = function(test, common) {
       data: data
     };
 
-    var expectedCount = 7;
+    var expectedCount = 9;
     dedupe(req, res, function () {
       t.equal(res.data.length, expectedCount, 'results have fewer items than before');
       t.end();