From a52691c61ea635b3b782ca3dfd0eaa404df71e1e Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 7 Feb 2017 17:02:29 -0500 Subject: [PATCH 1/5] feat: search by postalcode --- helper/geojsonify_place_details.js | 1 + helper/placeTypes.js | 3 +- helper/type_mapping.js | 4 +- middleware/trimByGranularity.js | 1 + package.json | 4 +- query/search.js | 19 ++++- test/unit/fixture/search_fallback.js | 69 +++++++++++++++++++ .../search_fallback_postalcode_only.js | 68 ++++++++++++++++++ test/unit/helper/type_mapping.js | 2 +- test/unit/query/search.js | 19 +++++ test/unit/sanitizer/_layers.js | 6 +- 11 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 test/unit/fixture/search_fallback_postalcode_only.js diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js index 44e827d5..d9a000ee 100644 --- a/helper/geojsonify_place_details.js +++ b/helper/geojsonify_place_details.js @@ -7,6 +7,7 @@ var DETAILS_PROPS = [ { name: 'housenumber', type: 'string' }, { name: 'street', type: 'string' }, { name: 'postalcode', type: 'string' }, + { name: 'postalcode_gid', type: 'string' }, { name: 'confidence', type: 'default' }, { name: 'match_type', type: 'string' }, { name: 'distance', type: 'default' }, diff --git a/helper/placeTypes.js b/helper/placeTypes.js index c9550ce7..defc9274 100644 --- a/helper/placeTypes.js +++ b/helper/placeTypes.js @@ -8,5 +8,6 @@ module.exports = [ 'localadmin', 'locality', 'borough', - 'neighbourhood' + 'neighbourhood', + 'postalcode' ]; diff --git a/helper/type_mapping.js b/helper/type_mapping.js index c43a96bb..7b9de5a0 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -51,7 +51,7 @@ var LAYERS_BY_SOURCE = { 'locality','borough', 'neighbourhood', 'venue' ], whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', - 'neighbourhood', 'microhood', 'disputed', 'venue'] + 'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode'] }; /* @@ -62,7 +62,7 @@ var LAYERS_BY_SOURCE = { var LAYER_ALIASES = { 'coarse': [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', - 'neighbourhood', 'microhood', 'disputed'] + 'neighbourhood', 'microhood', 'disputed', 'postalcode' ] }; // create a list of all layers by combining each entry from LAYERS_BY_SOURCE diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js index da4ef4f9..06bf9141 100644 --- a/middleware/trimByGranularity.js +++ b/middleware/trimByGranularity.js @@ -19,6 +19,7 @@ var layers = [ 'street', 'neighbourhood', 'borough', + 'postalcode', 'locality', 'localadmin', 'county', diff --git a/package.json b/package.json index 64ded821..12c6bbc7 100644 --- a/package.json +++ b/package.json @@ -55,8 +55,8 @@ "pelias-config": "2.7.1", "pelias-labels": "1.5.1", "pelias-logger": "0.1.0", - "pelias-model": "4.4.0", - "pelias-query": "8.12.0", + "pelias-model": "git://github.com/pelias/model.git#add-postalcode", + "pelias-query": "git://github.com/pelias/query.git#add-postalcodes", "pelias-text-analyzer": "1.7.0", "retry": "^0.10.1", "stats-lite": "2.0.3", diff --git a/query/search.js b/query/search.js index 48b47f31..91c22299 100644 --- a/query/search.js +++ b/query/search.js @@ -144,7 +144,10 @@ function generateQuery( clean ){ } function getQuery(vs) { - if (hasStreet(vs) || isCityStateOnlyWithOptionalCountry(vs) || isCityCountryOnly(vs)) { + if (hasStreet(vs) || + isCityStateOnlyWithOptionalCountry(vs) || + isCityCountryOnly(vs) || + isPostalCodeOnly(vs)) { return { type: 'fallback', body: fallbackQuery.render(vs) @@ -188,4 +191,18 @@ function isCityCountryOnly(vs) { } +function isPostalCodeOnly(vs) { + var isSet = (layer) => { + return vs.isset(`input:${layer}`); + }; + + var allowedFields = ['postcode']; + var disallowedFields = ['query', 'category', 'housenumber', 'street', + 'neighbourhood', 'borough', 'county', 'region', 'country']; + + return allowedFields.every(isSet) && + !disallowedFields.some(isSet); + +} + module.exports = generateQuery; diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js index 857e47e9..3835cadb 100644 --- a/test/unit/fixture/search_fallback.js +++ b/test/unit/fixture/search_fallback.js @@ -193,6 +193,75 @@ module.exports = { } } }, + { + 'bool': { + '_name': 'fallback.postalcode', + 'must': [ + { + 'multi_match': { + 'query': 'postalcode value', + 'type': 'phrase', + 'fields': [ + 'parent.postalcode' + ] + } + }, + { + 'multi_match': { + 'query': 'city value', + 'type': 'phrase', + 'fields': [ + 'parent.locality', + 'parent.locality_a', + 'parent.localadmin', + 'parent.localadmin_a' + ] + } + }, + { + 'multi_match': { + 'query': 'county value', + 'type': 'phrase', + 'fields': [ + 'parent.county', + 'parent.county_a', + 'parent.macrocounty', + 'parent.macrocounty_a' + ] + } + }, + { + 'multi_match': { + 'query': 'state value', + 'type': 'phrase', + 'fields': [ + 'parent.region', + 'parent.region_a', + 'parent.macroregion', + 'parent.macroregion_a' + ] + } + }, + { + 'multi_match': { + 'query': 'country value', + 'type': 'phrase', + 'fields': [ + 'parent.country', + 'parent.country_a', + 'parent.dependency', + 'parent.dependency_a' + ] + } + } + ], + 'filter': { + 'term': { + 'layer': 'postalcode' + } + } + } + }, { 'bool': { '_name': 'fallback.street', diff --git a/test/unit/fixture/search_fallback_postalcode_only.js b/test/unit/fixture/search_fallback_postalcode_only.js new file mode 100644 index 00000000..4519e166 --- /dev/null +++ b/test/unit/fixture/search_fallback_postalcode_only.js @@ -0,0 +1,68 @@ +module.exports = { + 'query': { + 'function_score': { + 'query': { + 'filtered': { + 'query': { + 'bool': { + 'should': [ + { + 'bool': { + '_name': 'fallback.postalcode', + 'must': [ + { + 'multi_match': { + 'query': '90210', + 'type': 'phrase', + 'fields': [ + 'parent.postalcode' + ] + } + } + ], + 'filter': { + 'term': { + 'layer': 'postalcode' + } + } + } + } + ] + } + }, + 'filter': { + 'bool': { + 'must': [] + } + } + } + }, + 'max_boost': 20, + 'functions': [ + { + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'popularity', + 'missing': 1 + }, + 'weight': 1 + }, + { + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'population', + 'missing': 1 + }, + 'weight': 2 + } + ], + 'score_mode': 'avg', + 'boost_mode': 'multiply' + } + }, + 'size': 20, + 'track_scores': true, + 'sort': [ + '_score' + ] +}; diff --git a/test/unit/helper/type_mapping.js b/test/unit/helper/type_mapping.js index a9ec4721..d67218c5 100644 --- a/test/unit/helper/type_mapping.js +++ b/test/unit/helper/type_mapping.js @@ -14,7 +14,7 @@ module.exports.tests.interfaces = function(test, common) { t.deepEquals(type_mapping.layer_mapping.coarse, [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', - 'borough', 'neighbourhood', 'microhood', 'disputed' ]); + 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ]); t.end(); }); diff --git a/test/unit/query/search.js b/test/unit/query/search.js index 1f312d0d..aa582082 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -603,6 +603,25 @@ module.exports.tests.city_country = function(test, common) { }); + test('valid postalcode only search', function(t) { + var clean = { + parsed_text: { + postalcode: '90210' + }, + text: '90210' + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_fallback_postalcode_only'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_fallback_postalcode_only'); + t.end(); + }); + + }; module.exports.all = function (tape, common) { diff --git a/test/unit/sanitizer/_layers.js b/test/unit/sanitizer/_layers.js index 5792e947..5a0c0693 100644 --- a/test/unit/sanitizer/_layers.js +++ b/test/unit/sanitizer/_layers.js @@ -43,7 +43,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var admin_layers = [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', - 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ]; + 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ]; t.deepEqual(clean.layers, admin_layers, 'coarse layers set'); t.end(); @@ -78,7 +78,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var expected_layers = [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', - 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ]; + 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode' ]; t.deepEqual(clean.layers, expected_layers, 'coarse + regular layers set'); t.end(); @@ -115,7 +115,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var coarse_layers = [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', 'neighbourhood', 'microhood', - 'disputed' ]; + 'disputed', 'postalcode' ]; var venue_layers = [ 'venue' ]; var expected_layers = venue_layers.concat(coarse_layers); From 6dd4cc9bd88d8dfd5a17d60a0567e621ea736833 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 9 Feb 2017 15:41:40 -0500 Subject: [PATCH 2/5] fix postalcodes not showing up in results --- middleware/normalizeParentIds.js | 4 +++ middleware/renamePlacenames.js | 45 +++++++++++++++++------- test/unit/middleware/renamePlacenames.js | 3 ++ 3 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 test/unit/middleware/renamePlacenames.js diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index e0e499e2..ae22a3c6 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -49,6 +49,10 @@ function normalizeParentIds(place) { * @return {string} */ function makeNewId(placeType, id) { + if (!id) { + return; + } + var doc = new Document('whosonfirst', placeType, id); return doc.getGid(); } diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index d89a43ad..9f63cb6e 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -1,12 +1,14 @@ -var _ = require('lodash'); +'use strict'; -var PARENT_PROPS = require('../helper/placeTypes'); +const _ = require('lodash'); -var ADDRESS_PROPS = { - 'number': 'housenumber', - 'zip': 'postalcode', - 'street': 'street' -}; +const PARENT_PROPS = require('../helper/placeTypes'); + +const ADDRESS_PROPS = [ + { name: 'number', newName: 'housenumber' }, + { name: 'zip', newName: 'postalcode', transform: (value) => { return [value]; } }, + { name: 'street', newName: 'street' } +]; function setup() { @@ -28,22 +30,39 @@ function renamePlacenames(req, res, next) { * Rename the fields in one record */ function renameOneRecord(place) { - if (place.address_parts) { - Object.keys(ADDRESS_PROPS).forEach(function (prop) { - place[ADDRESS_PROPS[prop]] = place.address_parts[prop]; - }); - } // merge the parent block into the top level object to flatten the structure + // only copy the properties if they have values if (place.parent) { - PARENT_PROPS.forEach(function (prop) { + PARENT_PROPS.forEach( (prop) => { place[prop] = place.parent[prop]; place[prop + '_a'] = place.parent[prop + '_a']; place[prop + '_gid'] = place.parent[prop + '_id']; }); } + // copy the address parts after parent hierarchy in order to prefer + // the postalcode specified by the original source data + if (place.address_parts) { + ADDRESS_PROPS.forEach( (prop) => { + renameAddressProperty(place, prop); + }); + } + return place; } +function renameAddressProperty(place, prop) { + if (!place.address_parts.hasOwnProperty(prop.name)) { + return; + } + + if (prop.hasOwnProperty('transform')) { + place[prop.newName] = prop.transform(place.address_parts[prop.name]); + } + else { + place[prop.newName] = place.address_parts[prop.name]; + } +} + module.exports = setup; diff --git a/test/unit/middleware/renamePlacenames.js b/test/unit/middleware/renamePlacenames.js new file mode 100644 index 00000000..15f296d2 --- /dev/null +++ b/test/unit/middleware/renamePlacenames.js @@ -0,0 +1,3 @@ +/** + * Created by diana on 2/8/17. + */ From 198d80ba5c2f66c564e34a19f5f261df1ff47245 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 9 Feb 2017 21:52:47 -0500 Subject: [PATCH 3/5] update pelias-model to the 4.5.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 12c6bbc7..3688535b 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "pelias-config": "2.7.1", "pelias-labels": "1.5.1", "pelias-logger": "0.1.0", - "pelias-model": "git://github.com/pelias/model.git#add-postalcode", + "pelias-model": "4.5.0", "pelias-query": "git://github.com/pelias/query.git#add-postalcodes", "pelias-text-analyzer": "1.7.0", "retry": "^0.10.1", From 587c1ede6cad5bb82f67b3a5a21f97e78cb76b37 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 9 Feb 2017 22:03:00 -0500 Subject: [PATCH 4/5] remove empty file that snuck in --- test/unit/middleware/renamePlacenames.js | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 test/unit/middleware/renamePlacenames.js diff --git a/test/unit/middleware/renamePlacenames.js b/test/unit/middleware/renamePlacenames.js deleted file mode 100644 index 15f296d2..00000000 --- a/test/unit/middleware/renamePlacenames.js +++ /dev/null @@ -1,3 +0,0 @@ -/** - * Created by diana on 2/8/17. - */ From 5a2942498c24fd59d0d4fe82e75fb045e8099089 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 7 Mar 2017 13:50:22 -0500 Subject: [PATCH 5/5] fix: update pelias-query to version with postalcode support --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7fa7e22..318bb8c4 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "pelias-labels": "1.5.1", "pelias-logger": "0.1.0", "pelias-model": "4.5.1", - "pelias-query": "8.12.0", + "pelias-query": "8.13.0", "pelias-text-analyzer": "1.7.2", "retry": "^0.10.1", "stats-lite": "2.0.3",