Merge pull request #756 from pelias/staging

Merge staging into production

controller/place.js

@@ -1,5 +1,5 @@
 var service = { mget: require('../service/mget') };
-var logger = require('pelias-logger').get('api:controller:place');
+var logger = require('pelias-logger').get('api');
 
 function setup( config, backend ){
 

controller/search.js

@@ -1,7 +1,7 @@
 var _ = require('lodash');
 
 var service = { search: require('../service/search') };
-var logger = require('pelias-logger').get('api:controller:search');
+var logger = require('pelias-logger').get('api');
 var logging = require( '../helper/logging' );
 
 function setup( config, backend, query ){

middleware/500.js

@@ -1,13 +1,19 @@
-var logger = require( 'pelias-logger' ).get( 'middleware-500' );
+var check = require('check-types'),
+    logger = require( 'pelias-logger' ).get( 'api' );
 
 // handle application errors
 function middleware(err, req, res, next) {
+
   logger.error( 'Error: `%s`. Stack trace: `%s`.', err, err.stack );
-  res.header('Cache-Control','public');
-  var error = (err && err.message) ? err.message : err;
 
-  if( res.statusCode < 400 ){ res.status(500); }
+  if( res.statusCode < 400 ){
-  res.json({ error: typeof error === 'string' ? error : 'internal server error' });
+    logger.info( 'status code changed from', res.statusCode, 'to 500' );
+    res.status(500);
+  }
+
+  var error = ( err && err.message ) ? err.message : err;
+  res.header('Cache-Control','public');
+  res.json({ error: check.nonEmptyString( error ) ? error : 'internal server error' });
 }
 
 module.exports = middleware;

middleware/confidenceScoreFallback.js

@@ -10,7 +10,7 @@
  */
 
 var check = require('check-types');
-var logger = require('pelias-logger').get('api-confidence');
+var logger = require('pelias-logger').get('api');
 
 function setup() {
   return computeScores;

middleware/sendJSON.js

@@ -1,5 +1,6 @@
 var check = require('check-types'),
     es = require('elasticsearch'),
+    logger = require( 'pelias-logger' ).get( 'api' ),
     exceptions = require('elasticsearch-exceptions/lib/exceptions/SupportedExceptions');
 
 // create a list of regular expressions to match against.
@@ -42,7 +43,10 @@ function sendJSONResponse(req, res, next) {
         if( err instanceof es.errors.RequestTimeout ){ statusCode = Math.max( statusCode, 408 ); }
         else if( err instanceof es.errors.NoConnections ){ statusCode = Math.max( statusCode, 502 ); }
         else if( err instanceof es.errors.ConnectionFault ){ statusCode = Math.max( statusCode, 502 ); }
-        else { statusCode = Math.max( statusCode, 500 ); }
+        else {
+          logger.warn( 'unknown geocoding error object:', err.constructor.name, err );
+          statusCode = Math.max( statusCode, 500 );
+        }
 
       /*
         some elasticsearch errors are only returned as strings (not instances of Error).
@@ -55,7 +59,10 @@ function sendJSONResponse(req, res, next) {
             statusCode = Math.max( statusCode, 500 );
             break; // break on first match
           }
+          logger.warn( 'unknown geocoding error string:', err );
         }
+      } else {
+        logger.warn( 'unknown geocoding error type:', typeof err, err );
       }
     });
   }

middleware/trimByGranularityStructured.js

@@ -22,8 +22,8 @@ const _ = require('lodash');
 //   supplied we want to retain borough=Manhattan and city=Manhattan results
 const layers = [
   'venue',
-  'street',
   'address',
+  'street',
   'neighbourhood',
   ['borough', 'locality'],
   'localadmin',
@@ -44,8 +44,8 @@ const layers = [
 // city=Manhattan
 const explicit_borough_layers = [
   'venue',
-  'street',
   'address',
+  'street',
   'neighbourhood',
   'borough',
   'locality',

package.json

@@ -52,14 +52,14 @@
     "markdown": "0.5.0",
     "morgan": "1.7.0",
     "pelias-categories": "1.1.0",
-    "pelias-config": "2.3.1",
+    "pelias-config": "2.4.0",
     "pelias-labels": "1.5.1",
     "pelias-logger": "0.1.0",
     "pelias-model": "4.3.0",
-    "pelias-query": "8.9.0",
+    "pelias-query": "8.11.0",
     "pelias-text-analyzer": "1.6.0",
     "stats-lite": "2.0.3",
-    "through2": "2.0.1"
+    "through2": "^2.0.3"
   },
   "devDependencies": {
     "ciao": "^0.6.0",

query/structured_geocoding.js

@@ -6,21 +6,21 @@ var peliasQuery = require('pelias-query'),
 //------------------------------
 // general-purpose search query
 //------------------------------
-var componentQuery = new peliasQuery.layout.ComponentFallbackQuery();
+var structuredQuery = new peliasQuery.layout.StructuredFallbackQuery();
 
 // scoring boost
-componentQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
+structuredQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
-componentQuery.score( peliasQuery.view.popularity_only_function );
+structuredQuery.score( peliasQuery.view.popularity_only_function );
-componentQuery.score( peliasQuery.view.population_only_function );
+structuredQuery.score( peliasQuery.view.population_only_function );
 // --------------------------------
 
 // non-scoring hard filters
-componentQuery.filter( peliasQuery.view.boundary_country );
+structuredQuery.filter( peliasQuery.view.boundary_country );
-componentQuery.filter( peliasQuery.view.boundary_circle );
+structuredQuery.filter( peliasQuery.view.boundary_circle );
-componentQuery.filter( peliasQuery.view.boundary_rect );
+structuredQuery.filter( peliasQuery.view.boundary_rect );
-componentQuery.filter( peliasQuery.view.sources );
+structuredQuery.filter( peliasQuery.view.sources );
-componentQuery.filter( peliasQuery.view.layers );
+structuredQuery.filter( peliasQuery.view.layers );
-componentQuery.filter( peliasQuery.view.categories );
+structuredQuery.filter( peliasQuery.view.categories );
 // --------------------------------
 
 /**
@@ -105,7 +105,7 @@ function generateQuery( clean ){
 function getQuery(vs) {
   return {
     type: 'fallback',
-    body: componentQuery.render(vs)
+    body: structuredQuery.render(vs)
   };
 }
 

query/text_parser_addressit.js

@@ -32,8 +32,6 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
   // ?
   else {
     logger.warn( 'chaos monkey asks: what happens now?' );
-    logger.warn( parsed_text );
-    try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
   }
 
   // ==== add parsed matches [address components] ====

routes/v1.js

@@ -8,7 +8,7 @@ var sanitizers = {
   place: require('../sanitizer/place'),
   search: require('../sanitizer/search'),
   search_fallback: require('../sanitizer/search_fallback'),
-  component_geocoding: require('../sanitizer/component_geocoding'),
+  structured_geocoding: require('../sanitizer/structured_geocoding'),
   reverse: require('../sanitizer/reverse'),
   nearby: require('../sanitizer/nearby')
 };
@@ -30,14 +30,14 @@ var controllers = {
 var queries = {
   libpostal: require('../query/search'),
   fallback_to_old_prod: require('../query/search_original'),
-  component_geocoding: require('../query/component_geocoding')
+  structured_geocoding: require('../query/structured_geocoding')
 };
 
 /** ----------------------- controllers ----------------------- **/
 
 var postProc = {
   trimByGranularity: require('../middleware/trimByGranularity'),
-  trimByGranularityComponent: require('../middleware/trimByGranularityComponent'),
+  trimByGranularityStructured: require('../middleware/trimByGranularityStructured'),
   distances: require('../middleware/distance'),
   confidenceScores: require('../middleware/confidenceScore'),
   confidenceScoresFallback: require('../middleware/confidenceScoreFallback'),
@@ -95,11 +95,11 @@ function addRoutes(app, peliasConfig) {
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
-    component: createRouter([
+    structured: createRouter([
-      sanitizers.component_geocoding.middleware,
+      sanitizers.structured_geocoding.middleware,
       middleware.calcSize(),
-      controllers.search(peliasConfig, undefined, queries.component_geocoding),
+      controllers.search(peliasConfig, undefined, queries.structured_geocoding),
-      postProc.trimByGranularityComponent(),
+      postProc.trimByGranularityStructured(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.confidenceScoresFallback(),
@@ -183,19 +183,19 @@ function addRoutes(app, peliasConfig) {
 
 
   // static data endpoints
-  app.get ( base,                  routers.index );
+  app.get ( base,                          routers.index );
-  app.get ( base + 'attribution',  routers.attribution );
+  app.get ( base + 'attribution',          routers.attribution );
-  app.get (        '/attribution', routers.attribution );
+  app.get (        '/attribution',         routers.attribution );
-  app.get (        '/status',      routers.status );
+  app.get (        '/status',              routers.status );
 
   // backend dependent endpoints
-  app.get ( base + 'place',        routers.place );
+  app.get ( base + 'place',                routers.place );
-  app.get ( base + 'autocomplete', routers.autocomplete );
+  app.get ( base + 'autocomplete',         routers.autocomplete );
-  app.get ( base + 'search',       routers.search );
+  app.get ( base + 'search',               routers.search );
-  app.post( base + 'search',       routers.search );
+  app.post( base + 'search',               routers.search );
-  app.get ( base + 'beta/component',    routers.component );
+  app.get ( base + 'search/structured',    routers.structured );
-  app.get ( base + 'reverse',      routers.reverse );
+  app.get ( base + 'reverse',              routers.reverse );
-  app.get ( base + 'nearby',       routers.nearby );
+  app.get ( base + 'nearby',               routers.nearby );
 
 }
 

sanitizer/_single_scalar_parameters.js

@@ -10,8 +10,10 @@ function sanitize( raw, clean ){
   Object.keys(raw).forEach(function(key) {
     if (_.isArray(raw[key])) {
       messages.errors.push('\'' + key + '\' parameter can only have one value');
+      delete raw[key];
     } else if (_.isObject(raw[key])) {
       messages.errors.push('\'' + key + '\' parameter must be a scalar');
+      delete raw[key];
     }
 
   });

sanitizer/_synthesize_analysis.js

@@ -1,4 +1,5 @@
 const _ = require('lodash');
+const text_analyzer = require('pelias-text-analyzer');
 
 const fields = {
   'address': 'address',
@@ -20,6 +21,19 @@ function isPostalCodeOnly(parsed_text) {
           parsed_text.hasOwnProperty('postalcode');
 }
 
+// figure out which field contains the probable house number, prefer number
+// libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street
+// so because we're treating the entire field as a street address, it's safe
+// to assume that an identified postcode is actually a house number.
+function getHouseNumberField(analyzed_address) {
+  for (var field of ['number', 'postalcode']) {
+    if (analyzed_address.hasOwnProperty(field)) {
+      return field;
+    }
+  }
+
+}
+
 function sanitize( raw, clean ){
 
   // error & warning messages
@@ -43,6 +57,34 @@ function sanitize( raw, clean ){
       `at least one of the following fields is required: ${Object.keys(fields).join(', ')}`);
   }
 
+  if (clean.parsed_text.hasOwnProperty('address')) {
+    const analyzed_address = text_analyzer.parse(clean.parsed_text.address);
+
+    const house_number_field = getHouseNumberField(analyzed_address);
+
+    // if we're fairly certain that libpostal identified a house number
+    // (from either the house_number or postcode field), place it into the
+    // number field and remove the first instance of that value from address
+    // and assign to street
+    // eg - '1090 N Charlotte St' becomes number=1090 and street=N Charlotte St
+    if (house_number_field) {
+      clean.parsed_text.number = analyzed_address[house_number_field];
+
+      // remove the first instance of the number and trim whitespace
+      clean.parsed_text.street = _.trim(_.replace(clean.parsed_text.address, clean.parsed_text.number, ''));
+
+    } else {
+      // otherwise no house number was identifiable, so treat the entire input
+      // as a street
+      clean.parsed_text.street = clean.parsed_text.address;
+
+    }
+
+    // the address field no longer means anything since it's been parsed, so remove it
+    delete clean.parsed_text.address;
+
+  }
+
   return messages;
 }
 

sanitizer/reverse.js

@@ -2,8 +2,8 @@
 var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitizer/sanitizeAll'),
     sanitizers = {
-      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitizers, must be run after them

sanitizer/search.js

@@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
 
 var sanitizeAll = require('../sanitizer/sanitizeAll'),
     sanitizers = {
-      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       text: require('../sanitizer/_text'),
       iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
       size: require('../sanitizer/_size')(/* use defaults*/),

sanitizer/search_fallback.js

@@ -4,7 +4,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'),
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
-var logger = require('pelias-logger').get('api:controller:search_fallback');
+var logger = require('pelias-logger').get('api');
 var logging = require( '../helper/logging' );
 
 // middleware

sanitizer/structured_geocoding.js

@@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
 
 var sanitizeAll = require('../sanitizer/sanitizeAll'),
     sanitizers = {
-      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
+      quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
       synthesize_analysis: require('../sanitizer/_synthesize_analysis'),
       iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
       size: require('../sanitizer/_size')(/* use defaults*/),

service/mget.js

@@ -11,7 +11,7 @@
 
 **/
 
-var peliasLogger = require( 'pelias-logger' ).get( 'service/mget' );
+var logger = require( 'pelias-logger' ).get( 'api' );
 
 function service( backend, query, cb ){
 
@@ -24,8 +24,17 @@ function service( backend, query, cb ){
 
   // query new backend
   backend().client.mget( cmd, function( err, data ){
+
+    // log total ms elasticsearch reported the query took to execute
+    if( data && data.took ){
+      logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
+    }
+
     // handle backend errors
-    if( err ){ return cb( err ); }
+    if( err ){
+      logger.error( 'backend error', err );
+      return cb( err );
+    }
 
     // map returned documents
     var docs = [];

service/search.js

@@ -5,18 +5,23 @@
 
 **/
 
-var peliasLogger = require( 'pelias-logger' ).get( 'service/search' );
+var logger = require( 'pelias-logger' ).get( 'api' );
 
 function service( backend, cmd, cb ){
 
   // query new backend
   backend().client.search( cmd, function( err, data ){
 
-    // handle backend errors
-    if( err ){ return cb( err ); }
-
     // log total ms elasticsearch reported the query took to execute
-    peliasLogger.verbose( 'time elasticsearch reported:', data.took / 1000 );
+    if( data && data.took ){
+      logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
+    }
+
+    // handle backend errors
+    if( err ){
+      logger.error( 'backend error', err );
+      return cb( err );
+    }
 
     // map returned documents
     var docs = [];

test/ciao/autocomplete/non_scalar_parameter_array.coffee

@@ -0,0 +1,30 @@
+
+#> define two sources
+path: '/v1/autocomplete?text=A&sources=A&sources=B'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

test/ciao/autocomplete/non_scalar_parameter_object.coffee

@@ -0,0 +1,30 @@
+
+#> define parameter as object
+path: '/v1/autocomplete?text=A&parameter[idx]=value'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

test/ciao/reverse/non_scalar_parameter_array.coffee

@@ -0,0 +1,30 @@
+
+#> define two sources
+path: '/v1/reverse?point.lat=1&point.lon=1&sources=A&sources=B'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

test/ciao/reverse/non_scalar_parameter_object.coffee

@@ -1,5 +1,5 @@
 
-#> set size
+#> define parameter as object
 path: '/v1/reverse?point.lat=1&point.lon=1&parameter[idx]=value'
 
 #? 200 ok

test/ciao/search/address_parsing.coffee

@@ -35,7 +35,7 @@ json.geocoding.query['size'].should.eql 10
 #? address parsing
 json.geocoding.query.parsed_text['number'].should.eql '30'
 json.geocoding.query.parsed_text['street'].should.eql 'w 26th st'
-json.geocoding.query.parsed_text['state'].should.eql 'ny'
+json.geocoding.query.parsed_text['state'].should.eql 'NY'
 
 json.features[0].properties.confidence.should.eql 1
 json.features[0].properties.match_type.should.eql "exact"

test/ciao/search/non_scalar_parameter_array.coffee

@@ -0,0 +1,30 @@
+
+#> define two sources
+path: '/v1/search?text=A&sources=A&sources=B'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

test/ciao/search/non_scalar_parameter_object.coffee

@@ -0,0 +1,30 @@
+
+#> define parameter as object
+path: '/v1/search?text=A&parameter[idx]=value'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

test/unit/fixture/structured_geocoding/fallback.json

@@ -6,6 +6,197 @@
           "query": {
             "bool": {
               "should": [
+                {
+                  "bool": {
+                    "_name": "fallback.address",
+                    "must": [
+                      {
+                        "match_phrase": {
+                          "address_parts.number": "number value"
+                        }
+                      },
+                      {
+                        "match_phrase": {
+                          "address_parts.street": "street value"
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "neighbourhood value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.neighbourhood",
+                            "parent.neighbourhood_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "borough value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.borough",
+                            "parent.borough_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "city value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.locality",
+                            "parent.locality_a",
+                            "parent.localadmin",
+                            "parent.localadmin_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "county value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.county",
+                            "parent.county_a",
+                            "parent.macrocounty",
+                            "parent.macrocounty_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "state value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.region",
+                            "parent.region_a",
+                            "parent.macroregion",
+                            "parent.macroregion_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "country value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.country",
+                            "parent.country_a",
+                            "parent.dependency",
+                            "parent.dependency_a"
+                          ]
+                        }
+                      }
+                    ],
+                    "should": [
+                      {
+                        "match_phrase": {
+                          "address_parts.zip": "postalcode value"
+                        }
+                      }
+                    ],
+                    "filter": {
+                      "term": {
+                        "layer": "address"
+                      }
+                    },
+                    "boost": 10
+                  }
+                },
+                {
+                  "bool": {
+                    "_name": "fallback.street",
+                    "must": [
+                      {
+                        "match_phrase": {
+                          "address_parts.street": "street value"
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "neighbourhood value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.neighbourhood",
+                            "parent.neighbourhood_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "borough value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.borough",
+                            "parent.borough_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "city value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.locality",
+                            "parent.locality_a",
+                            "parent.localadmin",
+                            "parent.localadmin_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "county value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.county",
+                            "parent.county_a",
+                            "parent.macrocounty",
+                            "parent.macrocounty_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "state value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.region",
+                            "parent.region_a",
+                            "parent.macroregion",
+                            "parent.macroregion_a"
+                          ]
+                        }
+                      },
+                      {
+                        "multi_match": {
+                          "query": "country value",
+                          "type": "phrase",
+                          "fields": [
+                            "parent.country",
+                            "parent.country_a",
+                            "parent.dependency",
+                            "parent.dependency_a"
+                          ]
+                        }
+                      }
+                    ],
+                    "should": [
+                      {
+                        "match_phrase": {
+                          "address_parts.zip": "postalcode value"
+                        }
+                      }
+                    ],
+                    "filter": {
+                      "term": {
+                        "layer": "street"
+                      }
+                    },
+                    "boost": 5
+                  }
+                },
                 {
                   "bool": {
                     "_name": "fallback.neighbourhood",
@@ -514,4 +705,4 @@
   ],
   "size": 20,
   "track_scores": true
-}
+}

test/unit/middleware/trimByGranularityStructured.js

@@ -1,4 +1,4 @@
-var trimByGranularity = require('../../../middleware/trimByGranularityComponent')();
+var trimByGranularity = require('../../../middleware/trimByGranularityStructured')();
 
 module.exports.tests = {};
 
@@ -19,8 +19,8 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'venue 1', _matched_queries: ['fallback.venue'] },
         { name: 'venue 2', _matched_queries: ['fallback.venue'] },
-        { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -55,9 +55,9 @@ module.exports.tests.trimByGranularity = function(test, common) {
 
     var res = {
       data: [
-        { name: 'street 1', _matched_queries: ['fallback.street'] },
-        { name: 'street 2', _matched_queries: ['fallback.street'] },
         { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'address 2', _matched_queries: ['fallback.address'] },
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -73,13 +73,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
     };
 
     var expected_data = [
-      { name: 'street 1', _matched_queries: ['fallback.street'] },
+      { name: 'address 1', _matched_queries: ['fallback.address'] },
-      { name: 'street 2', _matched_queries: ['fallback.street'] }
+      { name: 'address 2', _matched_queries: ['fallback.address'] }
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only street records should be here');
+        t.deepEquals(res.data, expected_data, 'only address records should be here');
         t.end();
       });
     }
@@ -87,13 +87,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
-  test('all records with fallback.* matched_queries name should retain only address when they are most granular', function(t) {
+  test('all records with fallback.* matched_queries name should retain only street when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
-        { name: 'address 2', _matched_queries: ['fallback.address'] },
+        { name: 'street 2', _matched_queries: ['fallback.street'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -109,13 +109,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
     };
 
     var expected_data = [
-      { name: 'address 1', _matched_queries: ['fallback.address'] },
+      { name: 'street 1', _matched_queries: ['fallback.street'] },
-      { name: 'address 2', _matched_queries: ['fallback.address'] },
+      { name: 'street 2', _matched_queries: ['fallback.street'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only address records should be here');
+        t.deepEquals(res.data, expected_data, 'only street records should be here');
         t.end();
       });
     }

test/unit/query/structured_geocoding.js

@@ -1,4 +1,4 @@
-var generate = require('../../../query/component_geocoding');
+var generate = require('../../../query/structured_geocoding');
 var fs = require('fs');
 
 module.exports.tests = {};
@@ -14,7 +14,6 @@ module.exports.tests.query = function(test, common) {
   test('valid search + focus + bbox', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test',
       querySize: 10,
@@ -29,7 +28,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_focus_bbox');
+    var expected = require('../fixture/structured_geocoding/linguistic_focus_bbox');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox');
@@ -39,7 +38,6 @@ module.exports.tests.query = function(test, common) {
   test('valid search + bbox', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test',
       querySize: 10,
@@ -53,7 +51,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_bbox');
+    var expected = require('../fixture/structured_geocoding/linguistic_bbox');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_bbox');
@@ -63,7 +61,6 @@ module.exports.tests.query = function(test, common) {
   test('valid lingustic-only search', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       layers: ['test']
@@ -72,7 +69,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_only');
+    var expected = require('../fixture/structured_geocoding/linguistic_only');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_only');
@@ -82,7 +79,6 @@ module.exports.tests.query = function(test, common) {
   test('search search + focus', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
@@ -92,7 +88,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_focus');
+    var expected = require('../fixture/structured_geocoding/linguistic_focus');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_focus');
@@ -102,7 +98,6 @@ module.exports.tests.query = function(test, common) {
   test('search search + viewport', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49136,
@@ -115,7 +110,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_viewport');
+    var expected = require('../fixture/structured_geocoding/linguistic_viewport');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_viewport');
@@ -127,7 +122,6 @@ module.exports.tests.query = function(test, common) {
   test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49135,
@@ -140,7 +134,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_viewport_min_diagonal');
+    var expected = require('../fixture/structured_geocoding/linguistic_viewport_min_diagonal');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'valid search query');
@@ -150,7 +144,6 @@ module.exports.tests.query = function(test, common) {
   test('search search + focus on null island', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       'focus.point.lat': 0, 'focus.point.lon': 0,
@@ -160,7 +153,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/linguistic_focus_null_island');
+    var expected = require('../fixture/structured_geocoding/linguistic_focus_null_island');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island');
@@ -187,7 +180,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse(JSON.stringify(query));
-    var expected = require('../fixture/component_geocoding/fallback');
+    var expected = require('../fixture/structured_geocoding/fallback');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'fallbackQuery');
@@ -198,7 +191,6 @@ module.exports.tests.query = function(test, common) {
   test('valid boundary.country search', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       text: 'test', querySize: 10,
       layers: ['test'],
@@ -208,7 +200,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/boundary_country');
+    var expected = require('../fixture/structured_geocoding/boundary_country');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search: valid boundary.country query');
@@ -218,7 +210,6 @@ module.exports.tests.query = function(test, common) {
   test('valid sources filter', function(t) {
     var clean = {
       parsed_text: {
-        street: 'street value'
       },
       'text': 'test',
       'sources': ['test_source']
@@ -227,7 +218,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/component_geocoding/with_source_filtering');
+    var expected = require('../fixture/structured_geocoding/with_source_filtering');
 
     t.deepEqual(compiled.type, 'fallback', 'query type set');
     t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering');
@@ -238,7 +229,7 @@ module.exports.tests.query = function(test, common) {
 
 module.exports.all = function (tape, common) {
   function test(name, testFunction) {
-    return tape('component_geocoding query ' + name, testFunction);
+    return tape('structured_geocoding query ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/run.js

@@ -30,15 +30,15 @@ var tests = [
   require('./middleware/sendJSON'),
   require('./middleware/normalizeParentIds'),
   require('./middleware/trimByGranularity'),
-  require('./middleware/trimByGranularityComponent'),
+  require('./middleware/trimByGranularityStructured'),
   require('./query/autocomplete'),
   require('./query/autocomplete_defaults'),
-  require('./query/component_geocoding'),
   require('./query/search_defaults'),
   require('./query/reverse_defaults'),
   require('./query/reverse'),
   require('./query/search'),
   require('./query/search_original'),
+  require('./query/structured_geocoding'),
   require('./query/text_parser'),
   require('./sanitizer/_boundary_country'),
   require('./sanitizer/_flag_bool'),
@@ -61,7 +61,7 @@ var tests = [
   require('./sanitizer/nearby'),
   require('./src/backend'),
   require('./sanitizer/autocomplete'),
-  require('./sanitizer/component_geocoding'),
+  require('./sanitizer/structured_geocoding'),
   require('./sanitizer/place'),
   require('./sanitizer/reverse'),
   require('./sanitizer/sanitizeAll'),

test/unit/sanitizer/_single_scalar_parameters.js

@@ -18,6 +18,11 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
       ],
       warnings: []
     });
+
+    // erroneous params should be deleted to avoid middleware errors
+    t.false(raw.arrayParameter1);
+    t.false(raw.arrayParameter2);
+
     t.end();
   });
 
@@ -36,6 +41,10 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
       ],
       warnings: []
     });
+
+    // erroneous params should be deleted to avoid middleware errors
+    t.false(raw.objectParameter1);
+    t.false(raw.objectParameter2);
     t.end();
   });
 

test/unit/sanitizer/_synthesize_analysis.js

@@ -1,13 +1,18 @@
-const sanitizer = require('../../../sanitizer/_synthesize_analysis');
 const _ = require('lodash');
+const proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
 
 module.exports.tests.text_parser = function(test, common) {
   test('all variables should be parsed', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.fail('parse should not have been called');
+      }
+    }});
+
     const raw = {
       query: ' \t query \t value \t ',
-      address: ' \t address \t value \t ',
       neighbourhood: ' \t neighbourhood \t value \t ',
       borough: ' \t borough \t value \t ',
       locality: ' \t locality \t value \t ',
@@ -21,7 +26,6 @@ module.exports.tests.text_parser = function(test, common) {
 
     const expected_clean = {
       parsed_text: {
-        address: 'address value',
         neighbourhood: 'neighbourhood value',
         borough: 'borough value',
         city: 'locality value',
@@ -42,6 +46,12 @@ module.exports.tests.text_parser = function(test, common) {
   });
 
   test('non-string and blank string values should be treated as not supplied', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.fail('parse should not have been called');
+      }
+    }});
+
     // helper to return a random value that's considered invalid
     function getInvalidValue() {
       return _.sample([{}, [], false, '', ' \t ', 17, undefined]);
@@ -75,6 +85,12 @@ module.exports.tests.text_parser = function(test, common) {
   });
 
   test('no supplied fields should return error', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.fail('parse should not have been called');
+      }
+    }});
+
     const raw = {};
 
     const clean = {};
@@ -92,6 +108,12 @@ module.exports.tests.text_parser = function(test, common) {
   });
 
   test('postalcode-only parsed_text should return error', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.fail('parse should not have been called');
+      }
+    }});
+
     const raw = {
       postalcode: 'postalcode value'
     };
@@ -113,6 +135,102 @@ module.exports.tests.text_parser = function(test, common) {
 
   });
 
+  test('text_analyzer identifying house number should extract it and street', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.equals(query, 'Number Value Street Value Number Value');
+
+        return {
+          number: 'Number Value'
+        };
+      }
+    }});
+
+    const raw = {
+      address: 'Number Value Street Value Number Value'
+    };
+
+    const clean = {};
+
+    const expected_clean = {
+      parsed_text: {
+        number: 'Number Value',
+        street: 'Street Value Number Value'
+      }
+    };
+
+    const messages = sanitizer(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('text_analyzer identifying postalcode but not house number should assign to number and remove from address', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.equals(query, 'Number Value Street Value Number Value');
+
+        return {
+          postalcode: 'Number Value'
+        };
+      }
+    }});
+
+    const raw = {
+      address: 'Number Value Street Value Number Value'
+    };
+
+    const clean = {};
+
+    const expected_clean = {
+      parsed_text: {
+        number: 'Number Value',
+        street: 'Street Value Number Value'
+      }
+    };
+
+    const messages = sanitizer(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('text_analyzer not revealing possible number should move address to street', function(t) {
+    var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
+      'pelias-text-analyzer': { parse: function(query) {
+        t.equals(query, 'Street Value');
+
+        return {};
+      }
+    }});
+
+    const raw = {
+      address: 'Street Value'
+    };
+
+    const clean = {};
+
+    const expected_clean = {
+      parsed_text: {
+        street: 'Street Value'
+      }
+    };
+
+    const messages = sanitizer(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
 };
 
 module.exports.all = function (tape, common) {

test/unit/sanitizer/nearby.js

@@ -30,7 +30,7 @@ module.exports.tests.interface = function(test, common) {
 
 module.exports.tests.sanitizers = function(test, common) {
   test('check sanitizer list', function (t) {
-    var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
+    var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
       'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country', 'categories'];
     t.deepEqual(Object.keys(nearby.sanitizer_list), expected);
     t.end();

test/unit/sanitizer/reverse.js

@@ -36,7 +36,7 @@ module.exports.tests.interface = function(test, common) {
 
 module.exports.tests.sanitizers = function(test, common) {
   test('check sanitizer list', function (t) {
-    var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
+    var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
       'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country'];
     t.deepEqual(Object.keys(reverse.sanitizer_list), expected);
     t.end();

test/unit/sanitizer/search.js

@@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
     });
 
     var expected_sanitizers = [
-      '_deprecate_quattroshapes',
       '_single_scalar_parameters',
+      '_deprecate_quattroshapes',
       '_text',
       '_iso2_to_iso3',
       '_size',

test/unit/sanitizer/structured_geocoding.js

@@ -8,7 +8,7 @@ module.exports.tests.sanitize = function(test, common) {
 
     // rather than re-verify the functionality of all the sanitizers, this test just verifies that they
     //  were all called correctly
-    var search = proxyquire('../../../sanitizer/component_geocoding', {
+    var search = proxyquire('../../../sanitizer/structured_geocoding', {
       '../sanitizer/_deprecate_quattroshapes': function() {
         called_sanitizers.push('_deprecate_quattroshapes');
         return { errors: [], warnings: [] };
@@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
     });
 
     var expected_sanitizers = [
-      '_deprecate_quattroshapes',
       '_single_scalar_parameters',
+      '_deprecate_quattroshapes',
       '_synthesize_analysis',
       '_iso2_to_iso3',
       '_size',
@@ -109,7 +109,7 @@ module.exports.tests.sanitize = function(test, common) {
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /component ' + name, testFunction);
+    return tape('SANTIZE /structured ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){