From 4ed5296d79c34e32cbe22073599234832a69e180 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:04:54 -0400 Subject: [PATCH 1/4] Update middleware with new prop name --- middleware/confidenceScore.js | 25 +++++++++++++------------ middleware/dedupe.js | 10 +++++----- middleware/localNamingConventions.js | 10 +++++----- middleware/renamePlacenames.js | 4 ++-- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index a0382b25..df48b2b1 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -100,9 +100,10 @@ function checkForDealBreakers(req, hit) { return true; } - if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address) && - req.clean.parsed_text.postalcode !== hit.address.zip) { - logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')'); + if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) && + req.clean.parsed_text.postalcode !== hit.address_parts.zip) { + logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + + ' !== ' + hit.address_parts.zip + ')'); return true; } } @@ -155,8 +156,8 @@ function checkName(text, parsed_text, hit) { */ function checkQueryType(text, hit) { if (check.assigned(text) && check.assigned(text.number) && - (check.undefined(hit.address) || - (check.assigned(hit.address) && check.undefined(hit.address.number)))) { + (check.undefined(hit.address_parts) || + (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) { return 0; } return 1; @@ -206,10 +207,10 @@ function propMatch(textProp, hitProp, expectEnriched) { * @param {string} [text.state] * @param {string} [text.country] * @param {object} hit - * @param {object} [hit.address] - * @param {string|number} [hit.address.number] - * @param {string} [hit.address.street] - * @param {string|number} [hit.address.zip] + * @param {object} [hit.address_parts] + * @param {string|number} [hit.address_parts.number] + * @param {string} [hit.address_parts.street] + * @param {string|number} [hit.address_parts.zip] * @param {Array} [hit.parent.region_a] * @param {Array} [hit.parent.country_a] * @returns {number} @@ -219,9 +220,9 @@ function checkAddress(text, hit) { var res = 0; if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) { - res += propMatch(text.number, (hit.address ? hit.address.number : null), false); - res += propMatch(text.street, (hit.address ? hit.address.street : null), false); - res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true); + res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false); + res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false); + res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true); res += propMatch(text.state, hit.parent.region_a[0], true); res += propMatch(text.country, hit.parent.country_a[0], true); diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 9e0e5c6a..d9336ffe 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -54,12 +54,12 @@ function isDifferent(item1, item2) { propMatch(item1, item2, 'name'); } - if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) { - propMatch(item1.address, item2.address, 'number'); - propMatch(item1.address, item2.address, 'street'); - propMatch(item1.address, item2.address, 'zip'); + if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) { + propMatch(item1.address_parts, item2.address_parts, 'number'); + propMatch(item1.address_parts, item2.address_parts, 'street'); + propMatch(item1.address_parts, item2.address_parts, 'zip'); } - else if (item1.address !== item2.address) { + else if (item1.address_parts !== item2.address_parts) { throw new Error('different'); } } diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 6170cda2..c78288ce 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -14,9 +14,9 @@ function applyLocalNamingConventions(req, res, next) { res.data.filter(function(place){ // only relevant for German addresses if( place.parent.country_a.indexOf('DEU') === -1 ){ return false; } - if( !place.hasOwnProperty('address') ){ return false; } - if( !place.address.hasOwnProperty('number') ){ return false; } - if( !place.address.hasOwnProperty('street') ){ return false; } + if( !place.hasOwnProperty('address_parts') ){ return false; } + if( !place.address_parts.hasOwnProperty('number') ){ return false; } + if( !place.address_parts.hasOwnProperty('street') ){ return false; } return true; }) .forEach( flipNumberAndStreet ); @@ -27,8 +27,8 @@ function applyLocalNamingConventions(req, res, next) { // DE address should have the housenumber and street name flipped // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101' function flipNumberAndStreet(place) { - var standard = ( place.address.number + ' ' + place.address.street ), - flipped = ( place.address.street + ' ' + place.address.number ); + var standard = ( place.address_parts.number + ' ' + place.address_parts.street ), + flipped = ( place.address_parts.street + ' ' + place.address_parts.number ); // flip street name and housenumber if( place.name.default === standard ){ diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 8f171e24..5bb7df67 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -58,9 +58,9 @@ function renamePlacenames(req, res, next) { * Rename the fields in one record */ function renameOneRecord(place) { - if (place.address) { + if (place.address_parts) { Object.keys(ADDRESS_PROPS).forEach(function (prop) { - place[ADDRESS_PROPS[prop]] = place.address[prop]; + place[ADDRESS_PROPS[prop]] = place.address_parts[prop]; }); } From 0917a84988be59c5281c4411aa462f05155a4759 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:05:25 -0400 Subject: [PATCH 2/4] Update property names in query defaults --- query/autocomplete_defaults.js | 6 +++--- query/reverse_defaults.js | 6 +++--- query/search_defaults.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 7bb430cc..ba52a049 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'multiply', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 2000, 'admin:country_a:analyzer': 'standard', diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index c32b1c3d..306efaac 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'replace', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 3, 'admin:country_a:analyzer': 'standard', diff --git a/query/search_defaults.js b/query/search_defaults.js index e9a918cd..ea0dc87f 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'replace', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 20, 'admin:country_a:analyzer': 'standard', From 74467e91e636e02a20aa6fc7d1853eac0bd09132 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:05:57 -0400 Subject: [PATCH 3/4] Update tests and fixtures --- .../dedupe_elasticsearch_nonascii_results.js | 2 +- .../fixture/dedupe_elasticsearch_results.js | 22 +++++++++---------- test/unit/fixture/search_full_address.js | 6 ++--- test/unit/fixture/search_regions_address.js | 4 ++-- .../unit/middleware/localNamingConventions.js | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js index b2a6c383..437060b9 100644 --- a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js @@ -55,7 +55,7 @@ module.exports = [ 'name': { 'default': '万里加油站' }, - 'address': { + 'address_parts': { 'street': 'S308', 'postalcode': '312044' }, diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index a57a3a7f..3fb6f3f9 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -4,7 +4,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.039265 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -31,7 +31,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.039265 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -58,7 +58,7 @@ module.exports = [ 'lon': -76.23246, 'lat': 39.99288 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['West Lampeter'], 'region_a': ['PA'], @@ -85,7 +85,7 @@ module.exports = [ 'lon': -76.20746, 'lat': 40.03927 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -112,7 +112,7 @@ module.exports = [ 'lon': -76.232457, 'lat': 39.992877 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Lampeter'], @@ -139,7 +139,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.038987 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Smoketown'], @@ -166,7 +166,7 @@ module.exports = [ 'lon': -76.20746, 'lat': 40.03899 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Smoketown'], @@ -193,7 +193,7 @@ module.exports = [ 'lon': -94.167445, 'lat': 38.762788 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Missouri'], 'locality': ['Strasburg'], @@ -219,7 +219,7 @@ module.exports = [ 'lon': -78.36317, 'lat': 38.98445 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, @@ -245,7 +245,7 @@ module.exports = [ 'lon': -100.16516, 'lat': 46.13427 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, @@ -271,7 +271,7 @@ module.exports = [ 'lon': -81.532392, 'lat': 40.597578 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 8e9fbaa8..8a8290ab 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -77,7 +77,7 @@ module.exports = { } },{ 'match': { - 'address.number': { + 'address_parts.number': { 'query': '123', 'boost': vs['address:housenumber:boost'], 'analyzer': vs['address:housenumber:analyzer'] @@ -85,7 +85,7 @@ module.exports = { } }, { 'match': { - 'address.street': { + 'address_parts.street': { 'query': 'main st', 'boost': vs['address:street:boost'], 'analyzer': vs['address:street:analyzer'] @@ -93,7 +93,7 @@ module.exports = { } }, { 'match': { - 'address.zip': { + 'address_parts.zip': { 'query': '10010', 'boost': vs['address:postcode:boost'], 'analyzer': vs['address:postcode:analyzer'] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index 0c2ddd8f..e0c05f3c 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -77,7 +77,7 @@ module.exports = { } },{ 'match': { - 'address.number': { + 'address_parts.number': { 'query': '1', 'boost': vs['address:housenumber:boost'], 'analyzer': vs['address:housenumber:analyzer'] @@ -85,7 +85,7 @@ module.exports = { } }, { 'match': { - 'address.street': { + 'address_parts.street': { 'query': 'water st', 'boost': vs['address:street:boost'], 'analyzer': vs['address:street:analyzer'] diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index d8714af5..d22642db 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -10,7 +10,7 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { '_type': 'test', 'name': { 'default': '1 Main St' }, 'center_point': { 'lon': -7.131521, 'lat': 54.428866 }, - 'address': { + 'address_parts': { 'zip': 'BT77 0BG', 'number': '1', 'street': 'Main St' @@ -27,7 +27,7 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { '_type': 'test', 'name': { 'default': '23 Grolmanstraße' }, 'center_point': { 'lon': 13.321487, 'lat': 52.506781 }, - 'address': { + 'address_parts': { 'zip': '10623', 'number': '23', 'street': 'Grolmanstraße' From 582fd50902b251f28988dfd8779bdb62adb35faa Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:06:23 -0400 Subject: [PATCH 4/4] Update pelias-query package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 919c19c5..49e00498 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", - "pelias-query": "^6.1.0", + "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", "through2": "2.0.1"