Merge pull request #796 from pelias/add-postalcodes

Search by postalcode

helper/geojsonify_place_details.js

@@ -7,6 +7,7 @@ var DETAILS_PROPS = [
   { name: 'housenumber',       type: 'string' },
   { name: 'street',            type: 'string' },
   { name: 'postalcode',        type: 'string' },
+  { name: 'postalcode_gid',    type: 'string' },
   { name: 'confidence',        type: 'default' },
   { name: 'match_type',        type: 'string' },
   { name: 'distance',          type: 'default' },

helper/placeTypes.js

@@ -8,5 +8,6 @@ module.exports = [
   'localadmin',
   'locality',
   'borough',
-  'neighbourhood'
+  'neighbourhood',
+  'postalcode'
 ];

helper/type_mapping.js

@@ -51,7 +51,7 @@ var LAYERS_BY_SOURCE = {
   'locality','borough', 'neighbourhood', 'venue' ],
  whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region',
    'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
-   'neighbourhood', 'microhood', 'disputed', 'venue']
+   'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode']
 };
 
 /*
@@ -62,7 +62,7 @@ var LAYERS_BY_SOURCE = {
 var LAYER_ALIASES = {
   'coarse': [ 'continent', 'country', 'dependency', 'macroregion', 'region',
    'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
-   'neighbourhood', 'microhood', 'disputed']
+   'neighbourhood', 'microhood', 'disputed', 'postalcode' ]
 };
 
 // create a list of all layers by combining each entry from LAYERS_BY_SOURCE

middleware/normalizeParentIds.js

@@ -49,6 +49,10 @@ function normalizeParentIds(place) {
  * @return {string}
  */
 function makeNewId(placeType, id) {
+  if (!id) {
+    return;
+  }
+
   var doc = new Document('whosonfirst', placeType, id);
   return doc.getGid();
 }

middleware/renamePlacenames.js

@@ -1,12 +1,14 @@
-var _ = require('lodash');
+'use strict';
 
-var PARENT_PROPS = require('../helper/placeTypes');
+const _ = require('lodash');
 
-var ADDRESS_PROPS = {
+const PARENT_PROPS = require('../helper/placeTypes');
-  'number': 'housenumber',
+
-  'zip': 'postalcode',
+const ADDRESS_PROPS = [
-  'street': 'street'
+  { name: 'number', newName: 'housenumber' },
-};
+  { name: 'zip',    newName: 'postalcode', transform: (value) => { return [value]; } },
+  { name: 'street', newName: 'street' }
+];
 
 
 function setup() {
@@ -28,22 +30,39 @@ function renamePlacenames(req, res, next) {
  * Rename the fields in one record
  */
 function renameOneRecord(place) {
-  if (place.address_parts) {
-    Object.keys(ADDRESS_PROPS).forEach(function (prop) {
-      place[ADDRESS_PROPS[prop]] = place.address_parts[prop];
-    });
-  }
 
   // merge the parent block into the top level object to flatten the structure
+  // only copy the properties if they have values
   if (place.parent) {
-    PARENT_PROPS.forEach(function (prop) {
+    PARENT_PROPS.forEach( (prop) => {
       place[prop] = place.parent[prop];
       place[prop + '_a'] = place.parent[prop + '_a'];
       place[prop + '_gid'] = place.parent[prop + '_id'];
     });
   }
 
+  // copy the address parts after parent hierarchy in order to prefer
+  // the postalcode specified by the original source data
+  if (place.address_parts) {
+    ADDRESS_PROPS.forEach( (prop) => {
+      renameAddressProperty(place, prop);
+    });
+  }
+
   return place;
 }
 
+function renameAddressProperty(place, prop) {
+  if (!place.address_parts.hasOwnProperty(prop.name)) {
+    return;
+  }
+
+  if (prop.hasOwnProperty('transform')) {
+    place[prop.newName] = prop.transform(place.address_parts[prop.name]);
+  }
+  else {
+    place[prop.newName] = place.address_parts[prop.name];
+  }
+}
+
 module.exports = setup;

middleware/trimByGranularity.js

@@ -19,6 +19,7 @@ var layers = [
   'street',
   'neighbourhood',
   'borough',
+  'postalcode',
   'locality',
   'localadmin',
   'county',

package.json

@@ -56,7 +56,7 @@
     "pelias-labels": "1.5.1",
     "pelias-logger": "0.1.0",
     "pelias-model": "4.5.1",
-    "pelias-query": "8.12.0",
+    "pelias-query": "8.13.0",
     "pelias-text-analyzer": "1.7.2",
     "retry": "^0.10.1",
     "stats-lite": "2.0.3",

query/search.js

@@ -144,7 +144,10 @@ function generateQuery( clean ){
 }
 
 function getQuery(vs) {
-  if (hasStreet(vs) || isCityStateOnlyWithOptionalCountry(vs) || isCityCountryOnly(vs)) {
+  if (hasStreet(vs) ||
+      isCityStateOnlyWithOptionalCountry(vs) ||
+      isCityCountryOnly(vs) ||
+      isPostalCodeOnly(vs)) {
     return {
       type: 'fallback',
       body: fallbackQuery.render(vs)
@@ -188,4 +191,18 @@ function isCityCountryOnly(vs) {
 
 }
 
+function isPostalCodeOnly(vs) {
+  var isSet = (layer) => {
+    return vs.isset(`input:${layer}`);
+  };
+
+  var allowedFields = ['postcode'];
+  var disallowedFields = ['query', 'category', 'housenumber', 'street',
+    'neighbourhood', 'borough', 'county', 'region', 'country'];
+
+  return allowedFields.every(isSet) &&
+    !disallowedFields.some(isSet);
+
+}
+
 module.exports = generateQuery;

test/unit/fixture/search_fallback.js

@@ -193,6 +193,75 @@ module.exports = {
                     }
                   }
                 },
+                {
+                  'bool': {
+                    '_name': 'fallback.postalcode',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'postalcode value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.postalcode'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'postalcode'
+                      }
+                    }
+                  }
+                },
                 {
                   'bool': {
                     '_name': 'fallback.street',

test/unit/fixture/search_fallback_postalcode_only.js

@@ -0,0 +1,68 @@
+module.exports = {
+  'query': {
+    'function_score': {
+      'query': {
+        'filtered': {
+          'query': {
+            'bool': {
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.postalcode',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': '90210',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.postalcode'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'postalcode'
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+          },
+          'filter': {
+            'bool': {
+              'must': []
+            }
+          }
+        }
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
+        }
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'multiply'
+    }
+  },
+  'size': 20,
+  'track_scores': true,
+  'sort': [
+    '_score'
+  ]
+};

test/unit/helper/type_mapping.js

@@ -14,7 +14,7 @@ module.exports.tests.interfaces = function(test, common) {
     t.deepEquals(type_mapping.layer_mapping.coarse,
                  [ 'continent', 'country', 'dependency', 'macroregion',
                    'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood',
-                   'borough', 'neighbourhood', 'microhood', 'disputed' ]);
+                   'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ]);
     t.end();
   });
 

test/unit/query/search.js

@@ -603,6 +603,25 @@ module.exports.tests.city_country = function(test, common) {
 
   });
 
+  test('valid postalcode only search', function(t) {
+    var clean = {
+      parsed_text: {
+        postalcode: '90210'
+      },
+      text: '90210'
+    };
+
+    var query = generate(clean);
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_fallback_postalcode_only');
+
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_fallback_postalcode_only');
+    t.end();
+  });
+
+
 };
 
 module.exports.all = function (tape, common) {

test/unit/sanitizer/_layers.js

@@ -43,7 +43,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
 
     var admin_layers = [ 'continent', 'country', 'dependency',
     'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county',
-    'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ];
+    'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ];
 
     t.deepEqual(clean.layers, admin_layers, 'coarse layers set');
     t.end();
@@ -78,7 +78,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
 
     var expected_layers = [ 'continent', 'country', 'dependency',
     'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county',
-    'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ];
+    'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ];
 
     t.deepEqual(clean.layers, expected_layers, 'coarse + regular layers set');
     t.end();
@@ -115,7 +115,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var coarse_layers = [ 'continent',
       'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin',
       'macrocounty', 'county', 'macrohood', 'borough', 'neighbourhood', 'microhood',
-      'disputed' ];
+      'disputed', 'postalcode' ];
 
     var venue_layers = [ 'venue' ];
     var expected_layers = venue_layers.concat(coarse_layers);