From 435104842da7d235a54a08db353594850dc327b7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 3 Feb 2016 14:46:16 -0500 Subject: [PATCH 001/106] Fix failing Ciao tests They were expecting certain parts of the response to not be there when an error is returned, but that's overspecifying the result in my opinion. --- test/ciao/search/layers_invalid.coffee | 2 -- test/ciao/search/layers_mix_invalid_valid.coffee | 2 -- test/ciao/search/sources_invalid.coffee | 2 -- 3 files changed, 6 deletions(-) diff --git a/test/ciao/search/layers_invalid.coffee b/test/ciao/search/layers_invalid.coffee index ec9c58bd..1e8e65fc 100644 --- a/test/ciao/search/layers_invalid.coffee +++ b/test/ciao/search/layers_invalid.coffee @@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -should.not.exist json.geocoding.query['types'] -should.not.exist json.geocoding.query['type'] diff --git a/test/ciao/search/layers_mix_invalid_valid.coffee b/test/ciao/search/layers_mix_invalid_valid.coffee index fab29ef2..11a40ddc 100644 --- a/test/ciao/search/layers_mix_invalid_valid.coffee +++ b/test/ciao/search/layers_mix_invalid_valid.coffee @@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -should.not.exist json.geocoding.query['types'] -should.not.exist json.geocoding.query['type'] diff --git a/test/ciao/search/sources_invalid.coffee b/test/ciao/search/sources_invalid.coffee index 2114c9c0..1d8df87c 100644 --- a/test/ciao/search/sources_invalid.coffee +++ b/test/ciao/search/sources_invalid.coffee @@ -32,5 +32,3 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -should.not.exist json.geocoding.query['types'] -should.not.exist json.geocoding.query['type'] From 0146050620357dbdba67e56321078d81c9c387e0 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 3 Feb 2016 14:48:18 -0500 Subject: [PATCH 002/106] Ignore failures removing docs directory If the docs files aren't there, it's expected that removing them would fail, and the task should continue. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eb9a4b10..901f11bc 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", - "docs": "rm -r docs; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs" + "docs": "rm -r docs || true; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs" }, "repository": { "type": "git", From 0f121c5ffeb5e1b61043a0796155ff3d383ef602 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 3 Feb 2016 14:56:22 -0500 Subject: [PATCH 003/106] Move doc generation commands to script file They were getting too large already, and I want to add more. --- bin/generate-docs | 6 ++++++ package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 bin/generate-docs diff --git a/bin/generate-docs b/bin/generate-docs new file mode 100755 index 00000000..46fce4e6 --- /dev/null +++ b/bin/generate-docs @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +rm -r docs || true + +cd test/ciao +node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs diff --git a/package.json b/package.json index 901f11bc..e14b4d91 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", - "docs": "rm -r docs || true; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs" + "docs": "./bin/generate-docs" }, "repository": { "type": "git", From a129df11254352530c8cdfb4afa09ab31d56ee68 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 3 Feb 2016 14:57:02 -0500 Subject: [PATCH 004/106] Check pelias server is running before generating docs And generate a helpful error message if not. --- bin/generate-docs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/generate-docs b/bin/generate-docs index 46fce4e6..5b337e3e 100755 --- a/bin/generate-docs +++ b/bin/generate-docs @@ -2,5 +2,8 @@ rm -r docs || true +curl -s http://localhost:3100/v1 > /dev/null || die "Pelias server does not appear to be running \ +on http://localhost:3100, run npm start in another window before generating docs." + cd test/ciao node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs From 79cd746d710c7fa045c16227076977f46c1a18d7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 9 Feb 2016 15:51:10 -0500 Subject: [PATCH 005/106] Update TravisCI config Adds Node 4.x versions, and containerized builds (faster). Connects pelias/pelias#258 --- .travis.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2a4e06a..aff00086 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,16 @@ +sudo: false language: node_js -script: "npm run unit" node_js: - - "0.10" - - "0.12" -sudo: false \ No newline at end of file + - 0.10 + - 0.12 + - 4.0 + - 4.1 + - 4.2 +matrix: + allow_failures: + - node_js: 4.0 + - node_js: 4.1 + - node_js: 4.2 +env: + - TEST_SUITE=unit +script: "npm run $TEST_SUITE" From 00b76c6e81874433534f2745a8162762d48d8d4f Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Sat, 13 Feb 2016 06:36:06 -0800 Subject: [PATCH 006/106] chore(package): update geojson to version 0.3.0 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e14b4d91..430a3ae8 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "express": "^4.8.8", "express-http-proxy": "^0.6.0", "extend": "2.0.1", - "geojson": "^0.2.1", + "geojson": "^0.3.0", "geojson-extent": "^0.3.1", "geolib": "^2.0.18", "geopipes-elasticsearch-backend": "^0.2.0", From d31fae3bd2d252ec0f815ae4f9cf0afa56d9cda7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 23 Feb 2016 14:11:45 -0500 Subject: [PATCH 007/106] Use patched branch of tap-dot Our test printer has a bug where a zero-test suite will pass. Usually, and definitely for this repo, the only way a test suite will not have any tests is if there was an error in the test program before it even ran any tests ([example](https://travis-ci.org/pelias/api/jobs/110970031)). Until my PR or something similar is merged into tap-dot, we should use this branch. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e14b4d91..c2df3c1e 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "nsp": "^0.3.0", "precommit-hook": "^1.0.7", "proxyquire": "^1.4.0", - "tap-dot": "^1.0.0", + "tap-dot": "git://github.com/orangejulius/tap-dot#error-on-zero-tests", "tape": "^2.13.4" } } From 92aaf960c396223eabbaee6f7e1fbc3f8f828796 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 23 Feb 2016 14:21:48 -0500 Subject: [PATCH 008/106] Upgrade tap-dot to 1.0.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c2df3c1e..cb8b05c1 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "nsp": "^0.3.0", "precommit-hook": "^1.0.7", "proxyquire": "^1.4.0", - "tap-dot": "git://github.com/orangejulius/tap-dot#error-on-zero-tests", + "tap-dot": "1.0.4", "tape": "^2.13.4" } } From 899d520cd784fdd75228e520c8171f695876866b Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Wed, 24 Feb 2016 11:50:03 +0200 Subject: [PATCH 009/106] Debug log full queries and responses between Pelias and Elasticsearch --- controller/search.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controller/search.js b/controller/search.js index 309e7647..9fafdd20 100644 --- a/controller/search.js +++ b/controller/search.js @@ -30,6 +30,8 @@ function setup( backend, query ){ cmd.type = req.clean.type; } + logger.debug( '[ES req]', JSON.stringify(cmd) ); + // query backend service.search( backend, cmd, function( err, docs, meta ){ @@ -42,7 +44,7 @@ function setup( backend, query ){ res.data = docs; res.meta = meta; } - + logger.debug('[ES response]', JSON.stringify(docs)); next(); }); From e50bce47c741ad2e996981bd9ba18e6560cc7d15 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 24 Feb 2016 18:34:13 -0500 Subject: [PATCH 010/106] Upgrade precommit-hook to 3.0.0 It just handles linting better. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cb8b05c1..0e4a27c1 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "istanbul": "^0.3.13", "jshint": "^2.5.6", "nsp": "^0.3.0", - "precommit-hook": "^1.0.7", + "precommit-hook": "3.0.0", "proxyquire": "^1.4.0", "tap-dot": "1.0.4", "tape": "^2.13.4" From e8fb26ea8562e2710b05b5a38ffd39d84c96baae Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 24 Feb 2016 18:34:37 -0500 Subject: [PATCH 011/106] Add precommit-hook sections to package.json --- package.json | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 0e4a27c1..db37dceb 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,9 @@ "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", - "docs": "./bin/generate-docs" + "docs": "./bin/generate-docs", + "lint": "jshint .", + "validate": "npm ls" }, "repository": { "type": "git", @@ -66,5 +68,10 @@ "proxyquire": "^1.4.0", "tap-dot": "1.0.4", "tape": "^2.13.4" - } -} + }, + "pre-commit": [ + "lint", + "validate", + "test" + ] +} \ No newline at end of file From d8426b3ecb4552203c5a25cc9a570fd01429273d Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 24 Feb 2016 19:41:40 -0500 Subject: [PATCH 012/106] Remove out of date EXAMPLES.md file --- EXAMPLES.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 EXAMPLES.md diff --git a/EXAMPLES.md b/EXAMPLES.md deleted file mode 100644 index 913afea3..00000000 --- a/EXAMPLES.md +++ /dev/null @@ -1,9 +0,0 @@ - -#### Search: - -- Nearest Museums: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=museum&size=40 -- Nearest Hotels: http://pelias.mapzen.com/search?lat=51.533&lon=-0.0652&input=hotel&size=40 - -#### Autocomplete: - -- Local Neighborhoods: http://pelias.mapzen.com/suggest/nearby?lat=40.7259&lon=-73.9806&input=e&layers=neighborhood&size=40 \ No newline at end of file From 269dff9dd1479fb27bc76be759dac65ebe3e9bce Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 25 Feb 2016 10:54:29 +0200 Subject: [PATCH 013/106] Add query/response logging in debug mode to place controller, too --- controller/place.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controller/place.js b/controller/place.js index c4e78978..1aa22aeb 100644 --- a/controller/place.js +++ b/controller/place.js @@ -1,4 +1,5 @@ var service = { mget: require('../service/mget') }; +var logger = require('pelias-logger').get('api:controller:place'); function setup( backend ){ @@ -46,6 +47,8 @@ function setup( backend ){ }; }); + logger.debug( '[ES req]', JSON.stringify(query) ); + service.mget( backend, query, function( err, docs ) { // error handler @@ -56,6 +59,7 @@ function setup( backend ){ else { res.data = docs; } + logger.debug('[ES response]', JSON.stringify(docs)); next(); }); From 05cb582500c47029b7d40b98fa0a303f3470a9b7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 25 Feb 2016 12:07:04 -0500 Subject: [PATCH 014/106] Add TravisCI settings for compiling node 4 modules --- .travis.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aff00086..a7bb9f63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,24 @@ node_js: - 4.0 - 4.1 - 4.2 + - 4.3 + - 5.6 matrix: allow_failures: - node_js: 4.0 - node_js: 4.1 - node_js: 4.2 + - node_js: 4.3 + - node_js: 5.6 env: - - TEST_SUITE=unit + global: + - CXX=g++-4.8 + matrix: + - TEST_SUITE=units script: "npm run $TEST_SUITE" +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 From 1cb210164e4700e3952f754930259d55cb04a36b Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 25 Feb 2016 12:20:52 -0500 Subject: [PATCH 015/106] Fix test suite name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a7bb9f63..f042d3c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ env: global: - CXX=g++-4.8 matrix: - - TEST_SUITE=units + - TEST_SUITE=unit script: "npm run $TEST_SUITE" addons: apt: From b95e175bb5fd99c1a9e5f8236ce0989eeb7bd31c Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Tue, 9 Feb 2016 08:11:52 -0800 Subject: [PATCH 016/106] chore(package): update dependencies http://greenkeeper.io/ --- package.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 286fef15..d720ed80 100644 --- a/package.json +++ b/package.json @@ -36,42 +36,42 @@ }, "dependencies": { "addressit": "git://github.com/dianashk/addressit.git#temp", - "async": "^0.9.0", - "check-types": "^3.3.1", + "async": "^1.5.2", + "check-types": "^6.0.0", "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", "express": "^4.8.8", "express-http-proxy": "^0.6.0", - "extend": "2.0.1", + "extend": "3.0.0", "geojson": "^0.3.0", "geojson-extent": "^0.3.1", "geolib": "^2.0.18", "geopipes-elasticsearch-backend": "^0.2.0", "iso3166-1": "^0.2.3", - "lodash": "^3.10.1", + "lodash": "^4.3.0", "markdown": "0.5.0", - "morgan": "1.5.2", + "morgan": "1.6.1", "pelias-config": "^1.0.1", - "pelias-esclient": "0.0.25", + "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", "pelias-query": "^5.0.0", - "pelias-schema": "1.0.0", - "pelias-suggester-pipeline": "2.0.2", - "stats-lite": "^1.0.3", - "through2": "0.6.5" + "pelias-schema": "2.0.0", + "pelias-suggester-pipeline": "2.0.4", + "stats-lite": "^2.0.0", + "through2": "2.0.1" }, "devDependencies": { "ciao": "^0.6.0", - "istanbul": "^0.3.13", + "istanbul": "^0.4.2", "jshint": "^2.5.6", - "nsp": "^0.3.0", - "precommit-hook": "3.0.0", + "nsp": "^2.2.0", + "precommit-hook": "^3.0.0", "proxyquire": "^1.4.0", "tap-dot": "1.0.4", - "tape": "^2.13.4" + "tape": "^4.4.0" }, "pre-commit": [ "lint", "validate", "test" ] -} \ No newline at end of file +} From f4a5dd812b43d9ba3097ffd851c692f471c344e1 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 9 Feb 2016 11:31:29 -0500 Subject: [PATCH 017/106] Replace depricated Lodash function contains with includes See https://github.com/lodash/lodash/wiki/Changelog#jan-12-2016--diff--docs --- helper/adminFields.js | 2 +- helper/geojsonify.js | 10 +++++----- helper/labelGenerator.js | 2 +- sanitiser/_details.js | 2 +- sanitiser/_flag_bool.js | 2 +- sanitiser/_groups.js | 4 ++-- sanitiser/_ids.js | 6 +++--- sanitiser/_source.js | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/helper/adminFields.js b/helper/adminFields.js index af6ac432..31648bc4 100644 --- a/helper/adminFields.js +++ b/helper/adminFields.js @@ -32,7 +32,7 @@ function getAvailableAdminFields(schema, expectedFields, logger) { // check if expected fields are actually in current schema var available = expectedFields.filter(function (field) { - return _.contains( actualFields, field ); + return _.includes( actualFields, field ); }); if (available.length === 0) { diff --git a/helper/geojsonify.js b/helper/geojsonify.js index d01b6f36..d77792ee 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -35,10 +35,10 @@ function lookupSource(src) { */ function lookupLayer(src) { if (src._type === 'geoname') { - if (_.contains(src.category, 'admin')) { - if (_.contains(src.category, 'admin:city')) { return 'locality'; } - if (_.contains(src.category, 'admin:admin1')) { return 'region'; } - if (_.contains(src.category, 'admin:admin2')) { return 'county'; } + if (_.includes(src.category, 'admin')) { + if (_.includes(src.category, 'admin:city')) { return 'locality'; } + if (_.includes(src.category, 'admin:admin1')) { return 'region'; } + if (_.includes(src.category, 'admin:admin2')) { return 'county'; } return 'neighbourhood'; // this could also be 'local_admin' } @@ -46,7 +46,7 @@ function lookupLayer(src) { if (src.address) { return 'address'; } } - if (_.contains(type_mapping.types, src._type)) { + if (_.includes(type_mapping.types, src._type)) { return type_mapping.type_to_layer[src._type]; } diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 1938232d..422c3409 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -16,7 +16,7 @@ module.exports = function( record ){ var buildOutput = function(parts, schemaArr, record) { for (var i=0; i 0 ){ From cb7b38bc754b8ecf2781e654f90382d1201ab76a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 9 Feb 2016 11:32:48 -0500 Subject: [PATCH 018/106] Replace check-types depricated unemptyString with nonEmptyString See https://www.npmjs.com/package/check-types#what-changed-from-4x-to-5x --- helper/labelGenerator.js | 2 +- sanitiser/_boundary_country.js | 2 +- sanitiser/_categories.js | 2 +- sanitiser/_ids.js | 4 ++-- sanitiser/_source.js | 2 +- sanitiser/_targets.js | 4 ++-- sanitiser/_text.js | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 422c3409..96fe5d09 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -16,7 +16,7 @@ module.exports = function( record ){ var buildOutput = function(parts, schemaArr, record) { for (var i=0; i0'); } From 769f190eabde8a6951b2f229a0d96a74630067b4 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 9 Feb 2016 11:38:22 -0500 Subject: [PATCH 019/106] Replace depricated Lodash function unique with uniq See https://github.com/lodash/lodash/wiki/Changelog#jan-12-2016--diff--docs --- helper/labelGenerator.js | 2 +- sanitiser/_ids.js | 2 +- sanitiser/_targets.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 96fe5d09..ffc5bf3f 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -29,7 +29,7 @@ module.exports = function( record ){ } // de-dupe outputs - labelParts = _.unique( labelParts ); + labelParts = _.uniq( labelParts ); return labelParts.join(', ').trim(); }; \ No newline at end of file diff --git a/sanitiser/_ids.js b/sanitiser/_ids.js index df879b94..a4d135a0 100644 --- a/sanitiser/_ids.js +++ b/sanitiser/_ids.js @@ -72,7 +72,7 @@ function sanitize( raw, clean ){ var rawIds = raw.ids.split(','); // deduplicate - rawIds = _.unique(rawIds); + rawIds = _.uniq(rawIds); // ensure all elements are valid non-empty strings if (!rawIds.every(check.nonEmptyString)) { diff --git a/sanitiser/_targets.js b/sanitiser/_targets.js index 734bd919..550d3a0c 100644 --- a/sanitiser/_targets.js +++ b/sanitiser/_targets.js @@ -61,7 +61,7 @@ function sanitize( raw, clean, opts ) { }, []); // dedupe in case aliases expanded to common things or user typed in duplicates - clean.types[typesKey] = _.unique(clean.types[typesKey]); + clean.types[typesKey] = _.uniq(clean.types[typesKey]); } } From 4c924605f88881a8d3df8a272ac17dfefc66c65f Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 9 Feb 2016 13:56:30 -0500 Subject: [PATCH 020/106] Keep stats-lite at version 1 to avoid using ES6 features Once we are using Node 4+, we can upgrade. See ttps://www.npmjs.com/package/stats-lite --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d720ed80..0d3aa4d2 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "pelias-query": "^5.0.0", "pelias-schema": "2.0.0", "pelias-suggester-pipeline": "2.0.4", - "stats-lite": "^2.0.0", + "stats-lite": "1.0.3", "through2": "2.0.1" }, "devDependencies": { From 358cee5407ddd4a6ac03650d647d4455c8f6a53d Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Wed, 17 Feb 2016 00:47:34 -0800 Subject: [PATCH 021/106] chore(package): update lodash to version 4.5.0 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0d3aa4d2..327deaad 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "geolib": "^2.0.18", "geopipes-elasticsearch-backend": "^0.2.0", "iso3166-1": "^0.2.3", - "lodash": "^4.3.0", + "lodash": "^4.5.0", "markdown": "0.5.0", "morgan": "1.6.1", "pelias-config": "^1.0.1", From 46afbc09fdc02b7beeda78f56da7fc73585c6888 Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Thu, 18 Feb 2016 21:05:21 -0800 Subject: [PATCH 022/106] chore(package): update morgan to version 1.7.0 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 327deaad..7c12ea7d 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "iso3166-1": "^0.2.3", "lodash": "^4.5.0", "markdown": "0.5.0", - "morgan": "1.6.1", + "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", From 0c3e4de0addb48a94f04e399eec650cff5dee602 Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Mon, 29 Feb 2016 12:08:55 -0800 Subject: [PATCH 023/106] chore(package): update pelias-schema to version 3.0.0 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7c12ea7d..1d681bf8 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", "pelias-query": "^5.0.0", - "pelias-schema": "2.0.0", + "pelias-schema": "3.0.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", "through2": "2.0.1" From 515d4aeb941e5b57d084df95a2efcb8a8ceb230c Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 1 Mar 2016 10:34:06 -0500 Subject: [PATCH 024/106] extracted `getSchema` helper method --- helper/labelGenerator.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index ffc5bf3f..bbe688fe 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -4,15 +4,10 @@ var _ = require('lodash'), schemas = require('./labelSchema.json'); module.exports = function( record ){ + var schema = getSchema(record.country_a); var labelParts = [ record.name.default ]; - var schema = schemas.default; - - if (record.country_a && record.country_a.length && schemas[record.country_a]) { - schema = schemas[record.country_a]; - } - var buildOutput = function(parts, schemaArr, record) { for (var i=0; i Date: Tue, 1 Mar 2016 10:47:38 -0500 Subject: [PATCH 025/106] broke country-specific label tests out to own file for easier testing --- helper/labelGenerator.js | 6 +- test/unit/helper/labelGenerator_GBR.js | 70 ++++++ test/unit/helper/labelGenerator_SGP.js | 51 +++++ test/unit/helper/labelGenerator_SWE.js | 53 +++++ test/unit/helper/labelGenerator_USA.js | 115 ++++++++++ ...Generator.js => labelGenerator_default.js} | 201 ------------------ test/unit/run.js | 6 +- 7 files changed, 296 insertions(+), 206 deletions(-) create mode 100644 test/unit/helper/labelGenerator_GBR.js create mode 100644 test/unit/helper/labelGenerator_SGP.js create mode 100644 test/unit/helper/labelGenerator_SWE.js create mode 100644 test/unit/helper/labelGenerator_USA.js rename test/unit/helper/{labelGenerator.js => labelGenerator_default.js} (55%) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index bbe688fe..34e353cd 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -23,10 +23,8 @@ module.exports = function( record ){ labelParts = buildOutput(labelParts, schema[key], record); } - // de-dupe outputs - labelParts = _.uniq( labelParts ); - - return labelParts.join(', ').trim(); + // de-dupe, join, trim + return _.uniq( labelParts ).join(', ').trim(); }; function getSchema(country_a) { diff --git a/test/unit/helper/labelGenerator_GBR.js b/test/unit/helper/labelGenerator_GBR.js new file mode 100644 index 00000000..38ab3f91 --- /dev/null +++ b/test/unit/helper/labelGenerator_GBR.js @@ -0,0 +1,70 @@ + +var generator = require('../../../helper/labelGenerator'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof generator, 'function', 'valid function'); + t.end(); + }); +}; + +// GBR street address +module.exports.tests.one_main_street_uk = function(test, common) { + test('one main street uk', function(t) { + var doc = { + 'name': { 'default': '1 Main St' }, + 'housenumber': '1', + 'street': 'Main St', + 'postalcode': 'BT77 0BG', + 'country_a': 'GBR', + 'country': 'United Kingdom', + 'region': 'Dungannon' + }; + t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom'); + t.end(); + }); +}; + +// GBR venue +module.exports.tests.hackney_city_farm = function(test, common) { + test('hackney city farm', function(t) { + var doc = { + 'name': { 'default': 'Hackney City Farm' }, + 'country_a': 'GBR', + 'country': 'United Kingdom', + 'region': 'Hackney', + 'county': 'Greater London', + 'locality': 'London', + 'neighbourhood': 'Haggerston' + }; + t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London'); + t.end(); + }); +}; + +// GBR country +module.exports.tests.wales = function(test, common) { + test('wales', function(t) { + var doc = { + 'name': { 'default': 'Wales' }, + 'country_a': 'GBR', + 'country': 'United Kingdom', + 'region': 'Wales' + }; + t.equal(generator(doc),'Wales, United Kingdom'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('label generator (GBR): ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/labelGenerator_SGP.js b/test/unit/helper/labelGenerator_SGP.js new file mode 100644 index 00000000..92aca117 --- /dev/null +++ b/test/unit/helper/labelGenerator_SGP.js @@ -0,0 +1,51 @@ + +var generator = require('../../../helper/labelGenerator'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof generator, 'function', 'valid function'); + t.end(); + }); +}; + +// SGP region +module.exports.tests.north_west_singapore = function(test, common) { + test('north west singapore', function(t) { + var doc = { + 'name': { 'default': 'North West' }, + 'country_a': 'SGP', + 'country': 'Singapore', + 'region': 'North West' + }; + t.equal(generator(doc),'North West, Singapore'); + t.end(); + }); +}; + +// SGP venue +module.exports.tests.singapore_mcdonalds = function(test, common) { + test('singapore_mcdonalds', function(t) { + var doc = { + 'name': { 'default': 'McDonald\'s' }, + 'country_a': 'SGP', + 'country': 'Singapore', + 'region': 'Central Singapore', + 'locality': 'Singapore' + }; + t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('label generator: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/labelGenerator_SWE.js b/test/unit/helper/labelGenerator_SWE.js new file mode 100644 index 00000000..594ea0e5 --- /dev/null +++ b/test/unit/helper/labelGenerator_SWE.js @@ -0,0 +1,53 @@ + +var generator = require('../../../helper/labelGenerator'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof generator, 'function', 'valid function'); + t.end(); + }); +}; + +// SWE city +module.exports.tests.skane1 = function(test, common) { + test('skåne 1', function(t) { + var doc = { + 'name': { 'default': 'Malmö' }, + 'country_a': 'SWE', + 'country': 'Sweden', + 'region': 'Skåne', + 'county': 'Malmö' + }; + t.equal(generator(doc),'Malmö, Skåne, Sweden'); + t.end(); + }); +}; + +// SWE city +module.exports.tests.skane2 = function(test, common) { + test('skåne 2', function(t) { + var doc = { + 'name': { 'default': 'Malmö' }, + 'country_a': 'SWE', + 'country': 'Sweden', + 'region': 'Skåne', + 'county': 'Malmö', + 'locality': 'Malmö' + }; + t.equal(generator(doc),'Malmö, Skåne, Sweden'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('label generator: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/labelGenerator_USA.js b/test/unit/helper/labelGenerator_USA.js new file mode 100644 index 00000000..93715839 --- /dev/null +++ b/test/unit/helper/labelGenerator_USA.js @@ -0,0 +1,115 @@ + +var generator = require('../../../helper/labelGenerator'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof generator, 'function', 'valid function'); + t.end(); + }); +}; + +// major USA city +module.exports.tests.san_francisco = function(test, common) { + test('san francisco', function(t) { + var doc = { + 'name': { 'default': 'San Francisco' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'California', + 'region_a': 'CA', + 'county': 'San Francisco County', + 'locality': 'San Francisco' + }; + t.equal(generator(doc),'San Francisco, San Francisco County, CA'); + t.end(); + }); +}; + +// USA venue +module.exports.tests.nyc_office = function(test, common) { + test('30 West 26th Street', function(t) { + var doc = { + 'name': { 'default': '30 West 26th Street' }, + 'housenumber': '30', + 'street': 'West 26th Street', + 'postalcode': '10010', + 'country_a': 'USA', + 'country': 'United States', + 'region': 'New York', + 'region_a': 'NY', + 'county': 'New York County', + 'localadmin': 'Manhattan', + 'locality': 'New York', + 'neighbourhood': 'Flatiron District' + }; + t.equal(generator(doc),'30 West 26th Street, Manhattan, NY'); + t.end(); + }); +}; + +// USA NYC eatery +module.exports.tests.nyc_bakery = function(test, common) { + test('New York Bakery', function(t) { + var doc = { + 'name': { 'default': 'New York Bakery' }, + 'housenumber': '51 W', + 'street': '29th', + 'country_a': 'USA', + 'country': 'United States', + 'region': 'New York', + 'region_a': 'NY', + 'county': 'New York County', + 'localadmin': 'Manhattan', + 'locality': 'New York', + 'neighbourhood': 'Koreatown' + }; + t.equal(generator(doc),'New York Bakery, Manhattan, NY'); + t.end(); + }); +}; + +// USA SFC building +module.exports.tests.ferry_building = function(test, common) { + test('Ferry Building', function(t) { + var doc = { + 'name': { 'default': 'Ferry Building' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'California', + 'region_a': 'CA', + 'county': 'San Francisco County', + 'locality': 'San Francisco', + 'neighbourhood': 'Financial District' + }; + t.equal(generator(doc),'Ferry Building, San Francisco, CA'); + t.end(); + }); +}; + +// USA state +module.exports.tests.california = function(test, common) { + test('california', function(t) { + var doc = { + 'name': { 'default': 'California' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'California', + 'region_a': 'CA' + }; + t.equal(generator(doc),'California, CA'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('label generator: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/labelGenerator.js b/test/unit/helper/labelGenerator_default.js similarity index 55% rename from test/unit/helper/labelGenerator.js rename to test/unit/helper/labelGenerator_default.js index 9a382397..dc8ad02d 100644 --- a/test/unit/helper/labelGenerator.js +++ b/test/unit/helper/labelGenerator_default.js @@ -10,84 +10,6 @@ module.exports.tests.interface = function(test, common) { }); }; -// major USA city -module.exports.tests.san_francisco = function(test, common) { - test('san francisco', function(t) { - var doc = { - 'name': { 'default': 'San Francisco' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA', - 'county': 'San Francisco County', - 'locality': 'San Francisco' - }; - t.equal(generator(doc),'San Francisco, San Francisco County, CA'); - t.end(); - }); -}; - -// USA venue -module.exports.tests.nyc_office = function(test, common) { - test('30 West 26th Street', function(t) { - var doc = { - 'name': { 'default': '30 West 26th Street' }, - 'housenumber': '30', - 'street': 'West 26th Street', - 'postalcode': '10010', - 'country_a': 'USA', - 'country': 'United States', - 'region': 'New York', - 'region_a': 'NY', - 'county': 'New York County', - 'localadmin': 'Manhattan', - 'locality': 'New York', - 'neighbourhood': 'Flatiron District' - }; - t.equal(generator(doc),'30 West 26th Street, Manhattan, NY'); - t.end(); - }); -}; - -// USA NYC eatery -module.exports.tests.nyc_bakery = function(test, common) { - test('New York Bakery', function(t) { - var doc = { - 'name': { 'default': 'New York Bakery' }, - 'housenumber': '51 W', - 'street': '29th', - 'country_a': 'USA', - 'country': 'United States', - 'region': 'New York', - 'region_a': 'NY', - 'county': 'New York County', - 'localadmin': 'Manhattan', - 'locality': 'New York', - 'neighbourhood': 'Koreatown' - }; - t.equal(generator(doc),'New York Bakery, Manhattan, NY'); - t.end(); - }); -}; - -// USA SFC building -module.exports.tests.ferry_building = function(test, common) { - test('Ferry Building', function(t) { - var doc = { - 'name': { 'default': 'Ferry Building' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA', - 'county': 'San Francisco County', - 'locality': 'San Francisco', - 'neighbourhood': 'Financial District' - }; - t.equal(generator(doc),'Ferry Building, San Francisco, CA'); - t.end(); - }); -}; - // AUS state module.exports.tests.new_south_wales = function(test, common) { test('new south wales', function(t) { @@ -102,21 +24,6 @@ module.exports.tests.new_south_wales = function(test, common) { }); }; -// USA state -module.exports.tests.california = function(test, common) { - test('california', function(t) { - var doc = { - 'name': { 'default': 'California' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA' - }; - t.equal(generator(doc),'California, CA'); - t.end(); - }); -}; - // IND state module.exports.tests.west_bengal = function(test, common) { test('west bengal', function(t) { @@ -194,20 +101,6 @@ module.exports.tests.wellington_victoria = function(test, common) { }); }; -// SGP region -module.exports.tests.north_west_singapore = function(test, common) { - test('north west singapore', function(t) { - var doc = { - 'name': { 'default': 'North West' }, - 'country_a': 'SGP', - 'country': 'Singapore', - 'region': 'North West' - }; - t.equal(generator(doc),'North West, Singapore'); - t.end(); - }); -}; - // IRQ region module.exports.tests.arbil = function(test, common) { test('arbil', function(t) { @@ -236,71 +129,6 @@ module.exports.tests.madrid = function(test, common) { }); }; -// SWE city -module.exports.tests.skane1 = function(test, common) { - test('skåne 1', function(t) { - var doc = { - 'name': { 'default': 'Malmö' }, - 'country_a': 'SWE', - 'country': 'Sweden', - 'region': 'Skåne', - 'county': 'Malmö' - }; - t.equal(generator(doc),'Malmö, Skåne, Sweden'); - t.end(); - }); -}; - -// SWE city -module.exports.tests.skane2 = function(test, common) { - test('skåne 2', function(t) { - var doc = { - 'name': { 'default': 'Malmö' }, - 'country_a': 'SWE', - 'country': 'Sweden', - 'region': 'Skåne', - 'county': 'Malmö', - 'locality': 'Malmö' - }; - t.equal(generator(doc),'Malmö, Skåne, Sweden'); - t.end(); - }); -}; - -// GBR street address -module.exports.tests.one_main_street_uk = function(test, common) { - test('one main street uk', function(t) { - var doc = { - 'name': { 'default': '1 Main St' }, - 'housenumber': '1', - 'street': 'Main St', - 'postalcode': 'BT77 0BG', - 'country_a': 'GBR', - 'country': 'United Kingdom', - 'region': 'Dungannon' - }; - t.equal(generator(doc),'1 Main St, Dungannon, United Kingdom'); - t.end(); - }); -}; - -// GBR venue -module.exports.tests.hackney_city_farm = function(test, common) { - test('hackney city farm', function(t) { - var doc = { - 'name': { 'default': 'Hackney City Farm' }, - 'country_a': 'GBR', - 'country': 'United Kingdom', - 'region': 'Hackney', - 'county': 'Greater London', - 'locality': 'London', - 'neighbourhood': 'Haggerston' - }; - t.equal(generator(doc),'Hackney City Farm, Haggerston, Greater London'); - t.end(); - }); -}; - // DEU street address module.exports.tests.one_grolmanstrasse = function(test, common) { test('one grolmanstrasse', function(t) { @@ -334,20 +162,6 @@ module.exports.tests.new_zealand = function(test, common) { }); }; -// GBR country -module.exports.tests.wales = function(test, common) { - test('wales', function(t) { - var doc = { - 'name': { 'default': 'Wales' }, - 'country_a': 'GBR', - 'country': 'United Kingdom', - 'region': 'Wales' - }; - t.equal(generator(doc),'Wales, United Kingdom'); - t.end(); - }); -}; - // IRL country module.exports.tests.republic_of_ireland = function(test, common) { test('northern ireland', function(t) { @@ -362,21 +176,6 @@ module.exports.tests.republic_of_ireland = function(test, common) { }); }; -// SGP venue -module.exports.tests.singapore_mcdonalds = function(test, common) { - test('singapore_mcdonalds', function(t) { - var doc = { - 'name': { 'default': 'McDonald\'s' }, - 'country_a': 'SGP', - 'country': 'Singapore', - 'region': 'Central Singapore', - 'locality': 'Singapore' - }; - t.equal(generator(doc),'McDonald\'s, Central Singapore, Singapore'); - t.end(); - }); -}; - // THA province module.exports.tests.krabi_province = function(test, common) { test('Krabi Provence', function(t) { diff --git a/test/unit/run.js b/test/unit/run.js index 0510cda6..c600e0ad 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -7,7 +7,11 @@ var tests = [ require('./controller/place'), require('./controller/search'), require('./helper/geojsonify'), - require('./helper/labelGenerator'), + require('./helper/labelGenerator_default'), + require('./helper/labelGenerator_GBR'), + require('./helper/labelGenerator_SGP'), + require('./helper/labelGenerator_SWE'), + require('./helper/labelGenerator_USA'), require('./helper/labelSchema'), require('./helper/text_parser'), require('./helper/type_mapping'), From 3e11e6687c1d9b2fa2334f9b60e9c555294ba0cd Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 1 Mar 2016 10:56:51 -0500 Subject: [PATCH 026/106] added `country_a` to label for all USA results --- helper/labelSchema.json | 3 ++- test/unit/helper/geojsonify.js | 2 +- test/unit/helper/labelGenerator_USA.js | 10 +++++----- test/unit/helper/labelSchema.js | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/helper/labelSchema.json b/helper/labelSchema.json index 4b555790..2c2859ea 100644 --- a/helper/labelSchema.json +++ b/helper/labelSchema.json @@ -1,7 +1,8 @@ { "USA": { "local": ["localadmin", "locality", "neighbourhood", "county"], - "regional": ["region_a", "region", "country"] + "regional": ["region_a", "region"], + "country": ["country_a"] }, "GBR": { "local": ["neighbourhood", "county", "localadmin", "locality", "region"], diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index a66421de..3c969302 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -199,7 +199,7 @@ module.exports.tests.search = function(test, common) { 'gid': 'osm:venue:34633854', 'layer': 'venue', 'source': 'osm', - 'label': 'Empire State Building, Manhattan, NY', + 'label': 'Empire State Building, Manhattan, NY, USA', 'name': 'Empire State Building', 'country_a': 'USA', 'country': 'United States', diff --git a/test/unit/helper/labelGenerator_USA.js b/test/unit/helper/labelGenerator_USA.js index 93715839..5512661d 100644 --- a/test/unit/helper/labelGenerator_USA.js +++ b/test/unit/helper/labelGenerator_USA.js @@ -22,7 +22,7 @@ module.exports.tests.san_francisco = function(test, common) { 'county': 'San Francisco County', 'locality': 'San Francisco' }; - t.equal(generator(doc),'San Francisco, San Francisco County, CA'); + t.equal(generator(doc),'San Francisco, San Francisco County, CA, USA'); t.end(); }); }; @@ -44,7 +44,7 @@ module.exports.tests.nyc_office = function(test, common) { 'locality': 'New York', 'neighbourhood': 'Flatiron District' }; - t.equal(generator(doc),'30 West 26th Street, Manhattan, NY'); + t.equal(generator(doc),'30 West 26th Street, Manhattan, NY, USA'); t.end(); }); }; @@ -65,7 +65,7 @@ module.exports.tests.nyc_bakery = function(test, common) { 'locality': 'New York', 'neighbourhood': 'Koreatown' }; - t.equal(generator(doc),'New York Bakery, Manhattan, NY'); + t.equal(generator(doc),'New York Bakery, Manhattan, NY, USA'); t.end(); }); }; @@ -83,7 +83,7 @@ module.exports.tests.ferry_building = function(test, common) { 'locality': 'San Francisco', 'neighbourhood': 'Financial District' }; - t.equal(generator(doc),'Ferry Building, San Francisco, CA'); + t.equal(generator(doc),'Ferry Building, San Francisco, CA, USA'); t.end(); }); }; @@ -98,7 +98,7 @@ module.exports.tests.california = function(test, common) { 'region': 'California', 'region_a': 'CA' }; - t.equal(generator(doc),'California, CA'); + t.equal(generator(doc),'California, CA, USA'); t.end(); }); }; diff --git a/test/unit/helper/labelSchema.js b/test/unit/helper/labelSchema.js index dcc3cdfa..442b7f70 100644 --- a/test/unit/helper/labelSchema.js +++ b/test/unit/helper/labelSchema.js @@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) { }; module.exports.tests.valid = function(test, common) { - var valid_keys = ['localadmin', 'locality', 'neighbourhood', 'county', 'region_a', 'region', 'country']; + var valid_keys = ['localadmin', 'locality', 'neighbourhood', 'county', 'region_a', 'region', 'country', 'country_a']; var default_schema = { local: ['localadmin', 'locality', 'neighbourhood', 'county', 'region'], regional: ['country'] From 5456632cdfee211e89674376d23662155161263c Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 1 Mar 2016 13:47:03 -0500 Subject: [PATCH 027/106] Added flag to not include `default.name` in label when source=`geonames` or `whosonfirst` and layer=`region` This will stop, say, a search for "New Mexico" resulting in the label "New Mexico, NM, USA" --- helper/labelGenerator.js | 11 ++++++- test/unit/helper/labelGenerator_USA.js | 44 +++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 34e353cd..3e4a4237 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -6,7 +6,7 @@ var _ = require('lodash'), module.exports = function( record ){ var schema = getSchema(record.country_a); - var labelParts = [ record.name.default ]; + var labelParts = getInitialLabel(record); var buildOutput = function(parts, schemaArr, record) { for (var i=0; i Date: Tue, 1 Mar 2016 15:57:58 -0500 Subject: [PATCH 028/106] added tests for USA local and regional fallbacks for label generation --- test/unit/helper/labelGenerator_USA.js | 181 ++++++++++++++++++------- 1 file changed, 130 insertions(+), 51 deletions(-) diff --git a/test/unit/helper/labelGenerator_USA.js b/test/unit/helper/labelGenerator_USA.js index 5a7ac532..8d994481 100644 --- a/test/unit/helper/labelGenerator_USA.js +++ b/test/unit/helper/labelGenerator_USA.js @@ -10,6 +10,136 @@ module.exports.tests.interface = function(test, common) { }); }; +module.exports.tests.localadmin = function(test, common) { + test('localadmin should trump locality, neighbourhood, and county', function(t) { + var doc = { + 'name': { 'default': 'Default Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'county': 'County Name', + 'localadmin': 'LocalAdmin Name', + 'locality': 'Locality Name', + 'neighbourhood': 'Neighbourhood Name' + }; + t.equal(generator(doc),'Default Name, LocalAdmin Name, Region Abbr, USA'); + t.end(); + }); +}; + +module.exports.tests.locality = function(test, common) { + test('locality should trump neighbourhood and county when localadmin not available', function(t) { + var doc = { + 'name': { 'default': 'Default Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'county': 'County Name', + 'locality': 'Locality Name', + 'neighbourhood': 'Neighbourhood Name' + }; + t.equal(generator(doc),'Default Name, Locality Name, Region Abbr, USA'); + t.end(); + }); +}; + +module.exports.tests.neighbourhood = function(test, common) { + test('neighbourhood should trump county when neither localadmin nor locality', function(t) { + var doc = { + 'name': { 'default': 'Default Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'county': 'County Name', + 'neighbourhood': 'Neighbourhood Name' + }; + t.equal(generator(doc),'Default Name, Neighbourhood Name, Region Abbr, USA'); + t.end(); + }); +}; + +module.exports.tests.county = function(test, common) { + test('county should be used when localadmin, locality, and neighbourhood are not available', function(t) { + var doc = { + 'name': { 'default': 'Default Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'county': 'County Name' + }; + t.equal(generator(doc),'Default Name, County Name, Region Abbr, USA'); + t.end(); + }); +}; + +module.exports.tests.region = function(test, common) { + test('region should be used when region_a is not available', function(t) { + var doc = { + 'name': { 'default': 'Default Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name' + }; + t.equal(generator(doc),'Default Name, Region Name, USA'); + t.end(); + }); +}; + +// USA geonames state +module.exports.tests.region_geonames = function(test, common) { + test('default name should not be prepended when source=geonames and layer=region', function(t) { + var doc = { + 'name': { 'default': 'Region Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'source': 'geonames', + 'layer': 'region' + }; + t.equal(generator(doc),'Region Abbr, USA'); + t.end(); + }); +}; + +// USA whosonfirst state +module.exports.tests.region_whosonfirst = function(test, common) { + test('default name should not be prepended when source=whosonfirst and layer=region', function(t) { + var doc = { + 'name': { 'default': 'California' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'source': 'whosonfirst', + 'layer': 'region' + }; + t.equal(generator(doc),'Region Abbr, USA'); + t.end(); + }); +}; + +// USA non-geonames/whosonfirst state +module.exports.tests.region_other_source = function(test, common) { + test('default name should not be prepended when source=whosonfirst and layer=region', function(t) { + var doc = { + 'name': { 'default': 'Region Name' }, + 'country_a': 'USA', + 'country': 'United States', + 'region': 'Region Name', + 'region_a': 'Region Abbr', + 'source': 'not geonames or whosonfirst', + 'layer': 'region' + }; + t.equal(generator(doc),'Region Name, Region Abbr, USA'); + t.end(); + }); +}; + // major USA city module.exports.tests.san_francisco = function(test, common) { test('san francisco', function(t) { @@ -88,57 +218,6 @@ module.exports.tests.ferry_building = function(test, common) { }); }; -// USA geonames state -module.exports.tests.california_geonames = function(test, common) { - test('default name should not be prepended when source=geonames and layer=region', function(t) { - var doc = { - 'name': { 'default': 'California' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA', - 'source': 'geonames', - 'layer': 'region' - }; - t.equal(generator(doc),'CA, USA'); - t.end(); - }); -}; - -// USA whosonfirst state -module.exports.tests.california_whosonfirst = function(test, common) { - test('default name should not be prepended when source=whosonfirst and layer=region', function(t) { - var doc = { - 'name': { 'default': 'California' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA', - 'source': 'whosonfirst', - 'layer': 'region' - }; - t.equal(generator(doc),'CA, USA'); - t.end(); - }); -}; - -// USA non-geonames/whosonfirst state -module.exports.tests.california_other_source = function(test, common) { - test('default name should not be prepended when source=whosonfirst and layer=region', function(t) { - var doc = { - 'name': { 'default': 'California' }, - 'country_a': 'USA', - 'country': 'United States', - 'region': 'California', - 'region_a': 'CA', - 'source': 'not geonames or whosonfirst', - 'layer': 'region' - }; - t.equal(generator(doc),'California, CA, USA'); - t.end(); - }); -}; - module.exports.all = function (tape, common) { function test(name, testFunction) { From 7b5b9dee61b378ec1e99637ead4956e1eff02d3b Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 1 Mar 2016 17:04:43 -0500 Subject: [PATCH 029/106] added comments illustrating the subtleties of label generation --- helper/labelGenerator.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 3e4a4237..c9eff405 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -24,7 +24,28 @@ module.exports = function( record ){ } // de-dupe, join, trim + // NOTE: while it may seem odd to call `uniq` on the list of label parts, + // the effect is quite subtle. Take, for instance, a result for "Lancaster, PA" + // the pseudo-object is: + // { + // 'name': 'Lancaster', + // 'locality': 'Lancaster', + // 'region_a': 'PA', + // 'country_a': 'USA' + // } + // + // the code up to this point generates the label: + // `Lancaster, Lancaster, PA, USA` + // + // then the `unique` call reduces this to: + // `Lancaster, PA, USA` + // + // this code doesn't have the same effect in the case of a venue or address + // where the `name` field would contain the address or name of a point-of-interest + // + // Also see https://github.com/pelias/api/issues/429 for other ways that this is bad return _.uniq( labelParts ).join(', ').trim(); + }; function getSchema(country_a) { From 08f414b3d57f92d6d931b593f7560873edb0efe1 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 1 Mar 2016 20:31:51 -0500 Subject: [PATCH 030/106] fixed encoding error that came from master --- test/unit/helper/labelGenerator_SWE.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/helper/labelGenerator_SWE.js b/test/unit/helper/labelGenerator_SWE.js index 594ea0e5..edefc93a 100644 --- a/test/unit/helper/labelGenerator_SWE.js +++ b/test/unit/helper/labelGenerator_SWE.js @@ -34,7 +34,7 @@ module.exports.tests.skane2 = function(test, common) { 'country': 'Sweden', 'region': 'Skåne', 'county': 'Malmö', - 'locality': 'Malmö' + 'locality': 'Malmö' }; t.equal(generator(doc),'Malmö, Skåne, Sweden'); t.end(); From 431fe44f05a0d4d6a059ea1d1a6fc36c3e937b83 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Wed, 2 Mar 2016 10:05:09 -0500 Subject: [PATCH 031/106] converted labelSchema to .js with module.exports this is in preparation to introduce functions into the schema instead of strings --- helper/labelGenerator.js | 5 +++-- helper/labelSchema.js | 23 +++++++++++++++++++++++ helper/labelSchema.json | 23 ----------------------- test/unit/helper/labelSchema.js | 2 +- 4 files changed, 27 insertions(+), 26 deletions(-) create mode 100644 helper/labelSchema.js delete mode 100644 helper/labelSchema.json diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index c9eff405..31a04414 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -1,7 +1,7 @@ var _ = require('lodash'), check = require('check-types'), - schemas = require('./labelSchema.json'); + schemas = require('./labelSchema'); module.exports = function( record ){ var schema = getSchema(record.country_a); @@ -23,7 +23,6 @@ module.exports = function( record ){ labelParts = buildOutput(labelParts, schema[key], record); } - // de-dupe, join, trim // NOTE: while it may seem odd to call `uniq` on the list of label parts, // the effect is quite subtle. Take, for instance, a result for "Lancaster, PA" // the pseudo-object is: @@ -44,6 +43,8 @@ module.exports = function( record ){ // where the `name` field would contain the address or name of a point-of-interest // // Also see https://github.com/pelias/api/issues/429 for other ways that this is bad + // + // de-dupe, join, trim return _.uniq( labelParts ).join(', ').trim(); }; diff --git a/helper/labelSchema.js b/helper/labelSchema.js new file mode 100644 index 00000000..d37ffc15 --- /dev/null +++ b/helper/labelSchema.js @@ -0,0 +1,23 @@ +module.exports = { + 'USA': { + 'local': ['localadmin', 'locality', 'neighbourhood', 'county'], + 'regional': ['region_a', 'region'], + 'country': ['country_a'] + }, + 'GBR': { + 'local': ['neighbourhood', 'county', 'localadmin', 'locality', 'region'], + 'regional': ['county','country','region'] + }, + 'SGP': { + 'local': ['neighbourhood', 'region', 'county', 'localadmin', 'locality'], + 'regional': ['county','country','region'] + }, + 'SWE': { + 'local': ['neighbourhood', 'region', 'county', 'localadmin', 'locality'], + 'regional': ['country'] + }, + 'default': { + 'local': ['localadmin', 'locality', 'neighbourhood', 'county', 'region'], + 'regional': ['country'] + } +}; diff --git a/helper/labelSchema.json b/helper/labelSchema.json deleted file mode 100644 index 2c2859ea..00000000 --- a/helper/labelSchema.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "USA": { - "local": ["localadmin", "locality", "neighbourhood", "county"], - "regional": ["region_a", "region"], - "country": ["country_a"] - }, - "GBR": { - "local": ["neighbourhood", "county", "localadmin", "locality", "region"], - "regional": ["county","country","region"] - }, - "SGP": { - "local": ["neighbourhood", "region", "county", "localadmin", "locality"], - "regional": ["county","country","region"] - }, - "SWE": { - "local": ["neighbourhood", "region", "county", "localadmin", "locality"], - "regional": ["country"] - }, - "default": { - "local": ["localadmin", "locality", "neighbourhood", "county", "region"], - "regional": ["country"] - } -} diff --git a/test/unit/helper/labelSchema.js b/test/unit/helper/labelSchema.js index 442b7f70..cb2308cd 100644 --- a/test/unit/helper/labelSchema.js +++ b/test/unit/helper/labelSchema.js @@ -1,5 +1,5 @@ -var schemas = require('../../../helper/labelSchema.json'); +var schemas = require('../../../helper/labelSchema'); var alpha3 = require('../mock/alpha3.json'); module.exports.tests = {}; From 0f6059c05a8a443376c8a39bca030e289733f304 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Thu, 3 Mar 2016 10:04:28 -0500 Subject: [PATCH 032/106] Refactored to use a function for looking up value for a label The reason for this switch is that special case logic is needed to determine the US state name to display. All other fields can be looked up by name. The burden of fallback functionality is now in `helper/labelSchema` so the burden of unit testing is now in `test/unit/helper/labelSchema.js` --- helper/labelGenerator.js | 17 +- helper/labelSchema.js | 65 ++- test/unit/helper/labelGenerator_USA.js | 13 +- test/unit/helper/labelSchema.js | 661 +++++++++++++++++++++++-- 4 files changed, 695 insertions(+), 61 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 31a04414..e772ceb9 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -1,6 +1,5 @@ var _ = require('lodash'), - check = require('check-types'), schemas = require('./labelSchema'); module.exports = function( record ){ @@ -8,19 +7,11 @@ module.exports = function( record ){ var labelParts = getInitialLabel(record); - var buildOutput = function(parts, schemaArr, record) { - for (var i=0; i Date: Thu, 3 Mar 2016 12:55:47 -0500 Subject: [PATCH 033/106] added tests for `default` schema --- test/unit/helper/labelSchema.js | 129 ++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/test/unit/helper/labelSchema.js b/test/unit/helper/labelSchema.js index 3d079b73..edbf2b77 100644 --- a/test/unit/helper/labelSchema.js +++ b/test/unit/helper/labelSchema.js @@ -645,6 +645,135 @@ module.exports.tests.swe = function(test, common) { }; +module.exports.tests.default = function(test, common) { + test('default.local should use localadmin value over locality, neighbourhood, county, region', function(t) { + var record = { + neighbourhood: 'neighbourhood value', + county: 'county value', + localadmin: 'localadmin value', + locality: 'locality value', + region: 'region value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value', 'localadmin value']); + t.end(); + + }); + + test('default.local should use locality value over neighbourhood, county, region', function(t) { + var record = { + neighbourhood: 'neighbourhood value', + county: 'county value', + locality: 'locality value', + region: 'region value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value', 'locality value']); + t.end(); + + }); + + test('default.local should use neighbourhood value over county, region', function(t) { + var record = { + neighbourhood: 'neighbourhood value', + county: 'county value', + region: 'region value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value', 'neighbourhood value']); + t.end(); + + }); + + test('default.local should use county value over region', function(t) { + var record = { + county: 'county value', + region: 'region value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value', 'county value']); + t.end(); + + }); + + test('default.local should use region value when nothing else is available', function(t) { + var record = { + region: 'region value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value', 'region value']); + t.end(); + + }); + + test('default.local should not append anything when none of neighbourhood, region, county, localadmin, ' + + 'locality are available', function(t) { + var record = {}; + + var labelParts = ['initial value']; + + var f = schemas.default.local; + + t.deepEqual(f(record, labelParts), ['initial value']); + t.end(); + + }); + + test('default.regional should use country over region, region_a, or country_a', function(t) { + var record = { + region: 'region value', + region_a: 'region_a value', + country: 'country value', + country_a: 'country_a value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.regional; + + t.deepEqual(f(record, labelParts), ['initial value', 'country value']); + t.end(); + + }); + + test('default.regional should not append any value if country is not available', function(t) { + var record = { + region: 'region value', + region_a: 'region_a value', + country_a: 'country_a value' + }; + + var labelParts = ['initial value']; + + var f = schemas.default.regional; + + t.deepEqual(f(record, labelParts), ['initial value']); + t.end(); + + }); + +}; + module.exports.all = function (tape, common) { function test(name, testFunction) { From cdd00af6bc766ac3c0c94494b119242ff07db605 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 21 Jan 2016 15:29:31 -0500 Subject: [PATCH 034/106] Build responses with new source and layer fields There is no longer any ambiguity in the source field and the layer (which is stored as _type), so a lot of logic to compute them can simply read from the elasticsearch document. Hooray! --- helper/geojsonify.js | 21 +-------------------- test/unit/helper/geojsonify.js | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index d77792ee..b415b18f 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -25,8 +25,7 @@ var DETAILS_PROPS = [ function lookupSource(src) { - var sources = type_mapping.type_to_source; - return sources.hasOwnProperty(src._type) ? sources[src._type] : src._type; + return src.source; } /* @@ -34,24 +33,6 @@ function lookupSource(src) { * Elasticsearch document source allows a more specific layer name to be chosen */ function lookupLayer(src) { - if (src._type === 'geoname') { - if (_.includes(src.category, 'admin')) { - if (_.includes(src.category, 'admin:city')) { return 'locality'; } - if (_.includes(src.category, 'admin:admin1')) { return 'region'; } - if (_.includes(src.category, 'admin:admin2')) { return 'county'; } - return 'neighbourhood'; // this could also be 'local_admin' - } - - if (src.name) { return 'venue'; } - if (src.address) { return 'address'; } - } - - if (_.includes(type_mapping.types, src._type)) { - return type_mapping.type_to_layer[src._type]; - } - - logger.warn('[geojsonify]: could not map _type ', src._type); - return src._type; } diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 3c969302..a5311050 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -40,7 +40,8 @@ module.exports.tests.search = function(test, common) { var input = [ { '_id': 'id1', - '_type': 'type1', + '_type': 'layer1', + 'source': 'source1', 'center_point': { 'lat': 51.5337144, 'lon': -0.1069716 @@ -72,7 +73,8 @@ module.exports.tests.search = function(test, common) { }, { '_id': 'id2', - '_type': 'type2', + '_type': 'layer2', + 'source': 'source2', 'name': { 'default': 'Blues Cafe' }, @@ -97,7 +99,8 @@ module.exports.tests.search = function(test, common) { }, { '_id': '34633854', - '_type': 'osmway', + '_type': 'way', + 'source': 'osm', 'name': { 'default': 'Empire State Building' }, @@ -141,9 +144,9 @@ module.exports.tests.search = function(test, common) { }, 'properties': { 'id': 'id1', - 'gid': 'type1:type1:id1', - 'layer': 'type1', - 'source': 'type1', + 'gid': 'source1:layer1:id1', + 'layer': 'layer1', + 'source': 'source1', 'label': '\'Round Midnight Jazz and Blues Bar, test3, Angel', 'name': '\'Round Midnight Jazz and Blues Bar', 'country_a': 'GBR', @@ -170,9 +173,9 @@ module.exports.tests.search = function(test, common) { }, 'properties': { 'id': 'id2', - 'gid': 'type2:type2:id2', - 'layer': 'type2', - 'source': 'type2', + 'gid': 'source2:layer2:id2', + 'layer': 'layer2', + 'source': 'source2', 'label': 'Blues Cafe, test3, Smithfield', 'name': 'Blues Cafe', 'country_a': 'GBR', @@ -196,8 +199,8 @@ module.exports.tests.search = function(test, common) { }, 'properties': { 'id': '34633854', - 'gid': 'osm:venue:34633854', - 'layer': 'venue', + 'gid': 'osm:way:34633854', + 'layer': 'way', 'source': 'osm', 'label': 'Empire State Building, Manhattan, NY, USA', 'name': 'Empire State Building', From 9ec2cd41d943b0f7a845f166c16a4ae8c80d8a0d Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 21 Jan 2016 16:07:21 -0500 Subject: [PATCH 035/106] Remove unused file: sanitiser/_source.js It was replaced by the dual purpose sanitiser/_targets.js --- sanitiser/_source.js | 46 -------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 sanitiser/_source.js diff --git a/sanitiser/_source.js b/sanitiser/_source.js deleted file mode 100644 index 6a4661dd..00000000 --- a/sanitiser/_source.js +++ /dev/null @@ -1,46 +0,0 @@ - -var _ = require('lodash'), - check = require('check-types'), - sources_map = require( '../query/sources' ); - -var ALL_SOURCES = Object.keys(sources_map), - ALL_SOURCES_JOINED = ALL_SOURCES.join(','); - -function sanitize( raw, clean ) { - - // error & warning messages - var messages = { errors: [], warnings: [] }; - - // init clean.types (if not already init) - clean.types = clean.types || {}; - - // default case (no layers specified in GET params) - // don't even set the from_layers key in this case - if( check.nonEmptyString( raw.source ) ){ - - var sources = raw.source.split(','); - - var invalid_sources = sources.filter(function(source) { - return !_.includes( ALL_SOURCES, source ); - }); - - if( invalid_sources.length > 0 ){ - invalid_sources.forEach( function( invalid ){ - messages.errors.push('\'' + invalid + '\' is an invalid source parameter. Valid options: ' + ALL_SOURCES_JOINED); - }); - } - - else { - var types = sources.reduce(function(acc, source) { - return acc.concat(sources_map[source]); - }, []); - - clean.types.from_source = types; - } - - } - - return messages; -} - -module.exports = sanitize; From fa07ba9ddcd26b6a1666022b90473b5acf76d39a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 21 Jan 2016 16:43:10 -0500 Subject: [PATCH 036/106] Add much needed whitespace --- test/unit/sanitiser/_layers.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/unit/sanitiser/_layers.js b/test/unit/sanitiser/_layers.js index 519d95d9..16ca19ae 100644 --- a/test/unit/sanitiser/_layers.js +++ b/test/unit/sanitiser/_layers.js @@ -10,6 +10,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.equal(messages.errors.length, 0, 'no errors'); t.end(); }); + test('invalid layer', function(t) { var raw = { layers: 'test_layer' }; var clean = {}; @@ -22,6 +23,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.true(messages.errors[0].length > msg.length, 'invalid error message'); t.end(); }); + test('venue (alias) layer', function(t) { var venue_layers = ['geoname','osmnode','osmway']; var raw = { layers: 'venue' }; @@ -32,6 +34,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers set'); t.end(); }); + test('coarse (alias) layer', function(t) { var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin']; var raw = { layers: 'coarse' }; @@ -42,6 +45,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, admin_layers, 'coarse layers set'); t.end(); }); + test('address (alias) layer', function(t) { var address_layers = ['osmaddress','openaddresses']; var raw = { layers: 'address' }; @@ -52,6 +56,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, address_layers, 'address layers set'); t.end(); }); + test('venue alias layer plus regular layers', function(t) { var venue_layers = ['geoname','osmnode','osmway']; var reg_layers = ['admin0', 'admin1']; @@ -63,6 +68,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, venue_layers.concat(reg_layers), 'venue + regular layers'); t.end(); }); + test('coarse alias layer plus regular layers', function(t) { var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin']; var reg_layers = ['geoname', 'osmnode', 'osmway']; @@ -75,6 +81,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, admin_layers.concat(reg_layers), 'coarse + regular layers set'); t.end(); }); + test('address alias layer plus regular layers', function(t) { var address_layers = ['osmaddress','openaddresses']; var reg_layers = ['admin0', 'locality']; @@ -87,6 +94,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, address_layers.concat(reg_layers), 'address + regular layers set'); t.end(); }); + test('alias layer plus regular layers (no duplicates)', function(t) { var venue_layers = ['geoname','osmnode','osmway','admin0']; var raw = { layers: 'venue,country' }; @@ -97,6 +105,7 @@ module.exports.tests.sanitize_layers = function(test, common) { t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers found (no duplicates)'); t.end(); }); + test('multiple alias layers (no duplicates)', function(t) { var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin']; var raw = { layers: 'venue,coarse' }; @@ -110,7 +119,6 @@ module.exports.tests.sanitize_layers = function(test, common) { }; module.exports.all = function (tape, common) { - function test(name, testFunction) { return tape('SANTIZE _layers ' + name, testFunction); } From 0ee1fdb4d946d7e572c754934cea2a51b2c8bce5 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 16 Feb 2016 17:11:51 -0500 Subject: [PATCH 037/106] Remove obsolete Geonames ID tests These are fixed now with our better source/layer mapping --- test/unit/sanitiser/_ids.js | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/test/unit/sanitiser/_ids.js b/test/unit/sanitiser/_ids.js index 40af368f..98237180 100644 --- a/test/unit/sanitiser/_ids.js +++ b/test/unit/sanitiser/_ids.js @@ -122,34 +122,6 @@ module.exports.tests.valid_ids = function(test, common) { }); }; -module.exports.tests.geonames = function(test, common) { - test('geonames venue maps correctly as normal', function(t) { - var raw = { ids: 'geonames:venue:15' }; - var clean = {}; - - var messages = sanitize( raw, clean); - - var expected_clean = { ids: [ { id: '15', types: [ 'geoname' ] } ] }; - t.deepEqual( messages.errors, [], 'no errors' ); - t.deepEqual( messages.warnings, [], 'no warnings' ); - t.deepEqual(clean, expected_clean, 'clean set correctly'); - t.end(); - }); - - test('arbitrary geonames layer maps to geoname type', function(t) { - var raw = { ids: 'geonames:address:16' }; // geonames technically has no address records! - var clean = {}; - - var messages = sanitize( raw, clean); - - var expected_clean = { ids: [ { id: '16', types: [ 'geoname' ] } ] }; - t.deepEqual( messages.errors, [], 'no errors' ); - t.deepEqual( messages.warnings, [], 'no warnings' ); - t.deepEqual(clean, expected_clean, 'clean set correctly'); - t.end(); - }); -}; - module.exports.tests.multiple_ids = function(test, common) { test('multiple ids', function(t) { var raw = { ids: 'geonames:venue:1,osm:venue:2' }; From 8fdb522f11f840cd050387e7abb0152895003781 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 24 Feb 2016 18:46:41 -0500 Subject: [PATCH 038/106] Upgrade to pelias-query 5.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1d681bf8..180576f6 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "pelias-config": "^1.0.1", "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", - "pelias-query": "^5.0.0", + "pelias-query": "^5.1.0", "pelias-schema": "3.0.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", From ef78b2e0fda3ae64f522e25b8c84ea80c2326994 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 22 Jan 2016 17:35:15 -0500 Subject: [PATCH 039/106] Filter source and layer by new fields This is functionally the same code as before, except that it uses the new source and layer mapping. --- controller/place.js | 30 +------ controller/search.js | 7 +- helper/text_parser.js | 2 +- helper/type_mapping.js | 126 +++++++++++++++--------------- helper/types.js | 43 ---------- middleware/_types.js | 40 ++-------- query/search.js | 4 +- sanitiser/_ids.js | 11 +-- sanitiser/_targets.js | 21 ++--- sanitiser/_text.js | 4 - sanitiser/reverse.js | 4 +- sanitiser/search.js | 4 +- service/mget.js | 1 - test/unit/controller/place.js | 8 +- test/unit/helper/text_parser.js | 2 +- test/unit/helper/type_mapping.js | 130 ++++--------------------------- test/unit/helper/types.js | 92 ---------------------- test/unit/run.js | 1 - test/unit/sanitiser/_ids.js | 47 +++++++---- test/unit/sanitiser/_layers.js | 54 +++++++------ test/unit/sanitiser/_sources.js | 31 ++++---- test/unit/sanitiser/_text.js | 1 - test/unit/sanitiser/place.js | 2 +- test/unit/sanitiser/reverse.js | 9 +-- test/unit/sanitiser/search.js | 5 +- 25 files changed, 197 insertions(+), 482 deletions(-) delete mode 100644 helper/types.js delete mode 100644 test/unit/helper/types.js diff --git a/controller/place.js b/controller/place.js index 1aa22aeb..29acfe8b 100644 --- a/controller/place.js +++ b/controller/place.js @@ -14,35 +14,10 @@ function setup( backend ){ return next(); } - /* req.clean.ids contains an array of objects with id and types properties. - * types is an array of one or more types, since it can't always be known which single - * type a gid might belong to (osmnode and osmway both have source osm and layer venue). - * - * However, the mget Elasticsearch query only accepts a single type at a - * time. - * - * So, first create a new array that, has an entry - * with each type and id combination. This requires creating a new array with more entries - * than req.clean.ids in the case where entries have multiple types. - */ - - var recordsToReturn = req.clean.ids.reduce(function (acc, ids_element) { - ids_element.types.forEach(function(type) { - acc.push({ - id: ids_element.id, - type: type - }); - }); - return acc; - }, []); - - /* - * Next, map the list of records to an Elasticsearch mget query - */ - var query = recordsToReturn.map( function(id) { + var query = req.clean.ids.map( function(id) { return { _index: 'pelias', - _type: id.type, + _type: id.layers, _id: id.id }; }); @@ -50,6 +25,7 @@ function setup( backend ){ logger.debug( '[ES req]', JSON.stringify(query) ); service.mget( backend, query, function( err, docs ) { + console.log('err:' + err); // error handler if( err ){ diff --git a/controller/search.js b/controller/search.js index 9fafdd20..4adaf31d 100644 --- a/controller/search.js +++ b/controller/search.js @@ -8,10 +8,10 @@ function setup( backend, query ){ query = query || require('../query/search'); function controller( req, res, next ){ - // do not run controller when a request // validation error has occurred. if( req.errors && req.errors.length ){ + delete req.clean.type; //to declutter output return next(); } @@ -25,9 +25,12 @@ function setup( backend, query ){ body: query( req.clean ) }; - // ? + // set the Elasticsearch types to filter by, + // and remove the property from clean so the API + // response output is cleaner if( req.clean.hasOwnProperty('type') ){ cmd.type = req.clean.type; + delete req.clean.type; } logger.debug( '[ES req]', JSON.stringify(cmd) ); diff --git a/helper/text_parser.js b/helper/text_parser.js index a561aef9..7b9ffcfe 100644 --- a/helper/text_parser.js +++ b/helper/text_parser.js @@ -15,7 +15,7 @@ module.exports = {}; module.exports.get_layers = function get_layers(query) { if (query.length <= 3 ) { // no address parsing required - return type_mapping.layer_with_aliases_to_type.coarse; + return type_mapping.layer_mapping.coarse; } }; diff --git a/helper/type_mapping.js b/helper/type_mapping.js index 06a8ec2d..ced5f1f3 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -1,87 +1,85 @@ var extend = require('extend'), _ = require('lodash'); -var TYPE_TO_SOURCE = { - 'geoname': 'gn', - 'osmnode': 'osm', - 'osmway': 'osm', - 'admin0': 'qs', - 'admin1': 'qs', - 'admin2': 'qs', - 'neighborhood': 'qs', - 'locality': 'qs', - 'local_admin': 'qs', - 'osmaddress': 'osm', - 'openaddresses': 'oa' -}; +function addStandardTargetsToAliases(standard, aliases) { + var combined = _.extend({}, aliases); + standard.forEach(function(target) { + if (combined[target] === undefined) { + combined[target] = [target]; + } + }); + + return combined; +} /* - * This doesn't include alias layers such as coarse + * Sources */ -var TYPE_TO_LAYER = { - 'geoname': 'venue', - 'osmnode': 'venue', - 'osmway': 'venue', - 'admin0': 'country', - 'admin1': 'region', - 'admin2': 'county', - 'neighborhood': 'neighbourhood', - 'locality': 'locality', - 'local_admin': 'localadmin', - 'osmaddress': 'address', - 'openaddresses': 'address' -}; -var SOURCE_TO_TYPE = { - 'gn' : ['geoname'], - 'geonames' : ['geoname'], - 'oa' : ['openaddresses'], - 'openaddresses' : ['openaddresses'], - 'qs' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], - 'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], - 'osm' : ['osmaddress', 'osmnode', 'osmway'], - 'openstreetmap' : ['osmaddress', 'osmnode', 'osmway'] +// a list of all sources +var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'quattroshapes', 'whosonfirst']; + +/* + * A list of alternate names for sources, mostly used to save typing + */ +var SOURCE_ALIASES = { + 'osm': ['openstreetmap'], + 'oa': ['openaddresses'], + 'gn': ['geonames'], + 'qs': ['quattroshapes'], + 'wof': ['whosonfirst'] }; -/** - * This does not included alias layers, those are built separately +/* + * Create an object that contains all sources or aliases. The key is the source or alias, + * the value is either that source, or the canonical name for that alias if it's an alias. */ -var LAYER_TO_TYPE = { - 'venue': ['geoname','osmnode','osmway'], - 'address': ['osmaddress','openaddresses'], - 'country': ['admin0'], - 'region': ['admin1'], - 'county': ['admin2'], - 'locality': ['locality'], - 'localadmin': ['local_admin'], - 'neighbourhood': ['neighborhood'] +var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES); + +/* + * Layers + */ + +/* + * A list of all layers in each source. This is used for convenience elswhere + * and to determine when a combination of source and layer parameters is + * not going to match any records and will return no results. + */ +var LAYERS_BY_SOURCE = { + openstreetmap: [ 'address', 'venue' ], + openaddresses: [ 'address' ], + geonames: [ 'country', 'region', 'county', 'locality', 'venue' ], + quattroshapes: ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], + whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region', + 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed'] }; +/* + * A list of layer aliases that can be used to support specific use cases + * (like coarse geocoding) * or work around the fact that different sources + * may have layers that mean the same thing but have a different name + */ var LAYER_ALIASES = { - 'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin'] + 'coarse': LAYERS_BY_SOURCE.whosonfirst.concat(LAYERS_BY_SOURCE.quattroshapes), + 'country': ['country', 'admin0'], // Include both QS and WOF layers for various types of places + 'region': ['region', 'admin1'] // Alias layers that include themselves look weird, but do work }; -var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE); +// create a list of all layers by combining each entry from LAYERS_BY_SOURCE +var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) { + return acc.concat(LAYERS_BY_SOURCE[key]); +}, [])); /* - * derive the list of types, sources, and layers from above mappings + * Create the an object that has a key for each possible layer or alias, + * and returns either that layer, or all the layers in the alias */ -var TYPES = Object.keys(TYPE_TO_SOURCE); -var SOURCES = Object.keys(SOURCE_TO_TYPE); -var LAYERS = Object.keys(LAYER_TO_TYPE); - -var sourceAndLayerToType = function sourceAndLayerToType(source, layer) { - return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]); -}; +var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES); module.exports = { - types: TYPES, sources: SOURCES, layers: LAYERS, - type_to_source: TYPE_TO_SOURCE, - type_to_layer: TYPE_TO_LAYER, - source_to_type: SOURCE_TO_TYPE, - layer_to_type: LAYER_TO_TYPE, - layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE, - source_and_layer_to_type: sourceAndLayerToType + source_mapping: SOURCE_MAPPING, + layer_mapping: LAYER_MAPPING, + layers_by_source: LAYERS_BY_SOURCE }; diff --git a/helper/types.js b/helper/types.js deleted file mode 100644 index d9dec59d..00000000 --- a/helper/types.js +++ /dev/null @@ -1,43 +0,0 @@ -var type_mapping = require( '../helper/type_mapping' ); -var _ = require('lodash'); - -/** - * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched - * This method decides how to combine all the preferences. - * - * @param {Array} clean_types - * @returns {Array} - */ -module.exports = function calculate_types(clean_types) { - //Check that at least one preference of types is defined - if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) { - throw new Error('clean_types should not be null or undefined'); - } - - /* the layers and source parameters are cumulative: - * perform a set intersection of their specified types - */ - if (clean_types.from_layers || clean_types.from_sources) { - var types = type_mapping.types; - - if (clean_types.from_layers) { - types = _.intersection(types, clean_types.from_layers); - } - - if (clean_types.from_sources) { - types = _.intersection(types, clean_types.from_sources); - } - - return types; - } - - /* - * Type restrictions requested by the address parser should only be used - * if both the source and layers parameters are empty, so do this last - */ - if (clean_types.from_text_parser) { - return clean_types.from_text_parser; - } - - throw new Error('no types specified'); -}; diff --git a/middleware/_types.js b/middleware/_types.js index 3e604f7b..ccabdab9 100644 --- a/middleware/_types.js +++ b/middleware/_types.js @@ -1,43 +1,15 @@ -var types_helper = require( '../helper/types' ); - /** - * Validate the types specified to be searched. + * Take the layers specified by the layers parameter and use them to set the + * list of Elasticsearch types to filter. * - * Elasticsearch interprets an empty array of types as "search anything" rather - * than "search nothing", so in the case of an empty array, return an error - * message instead of searching at all. + * This has to be done outside the layers sanitizer since it doesn't know that + * the layers property is eventualy used to choose the _type. */ function middleware(req, res, next) { req.clean = req.clean || {}; - if (req.clean.hasOwnProperty('types')) { - - try { - var types = types_helper(req.clean.types); - - if ((types instanceof Array) && types.length === 0) { - var err = 'You have specified both the `sources` and `layers` ' + - 'parameters in a combination that will return no results.'; - req.errors.push( err ); - } - - else { - req.clean.type = types; - } - - } - - // @todo: refactor this flow, it is confusing as `types_helper()` can throw - // with an error "clean_types should not be null or undefined" which is - // not returned to the user yet the return value CAN trigger a user error. - // I would have liked to throw for BOTH cases and then handle the users errors - // inside the 'catch' but this is not possible. - // also: why are we deleting things from $clean? - catch (err) { - // this means there were no types specified - delete req.clean.types; - } - + if (req.clean.hasOwnProperty('layers')) { + req.clean.type = req.clean.layers; } next(); diff --git a/query/search.js b/query/search.js index 764cf999..804cb88b 100644 --- a/query/search.js +++ b/query/search.js @@ -37,7 +37,7 @@ query.score( peliasQuery.view.admin('neighborhood') ); // non-scoring hard filters query.filter( peliasQuery.view.boundary_circle ); query.filter( peliasQuery.view.boundary_rect ); - +query.filter( peliasQuery.view.sources ); // -------------------------------- /** @@ -51,6 +51,8 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); + vs.var( 'sources', clean.sources); + // size if( clean.querySize ) { vs.var( 'size', clean.querySize ); diff --git a/sanitiser/_ids.js b/sanitiser/_ids.js index a4d135a0..4a1844be 100644 --- a/sanitiser/_ids.js +++ b/sanitiser/_ids.js @@ -45,17 +45,10 @@ function sanitizeId(rawId, messages) { return; } - //TODO: remove this once we have a better set of layers for Geonames - var types; - if (source === 'gn' || source === 'geonames') { - types = ['geoname']; - } else { - types = type_mapping.source_and_layer_to_type(source, layer); - } - return { + source: source, + layer: layer, id: id, - types: types }; } diff --git a/sanitiser/_targets.js b/sanitiser/_targets.js index 550d3a0c..a8a50400 100644 --- a/sanitiser/_targets.js +++ b/sanitiser/_targets.js @@ -1,25 +1,24 @@ - var _ = require('lodash'), check = require('check-types'); +function getValidKeys(mapping) { + return _.uniq(Object.keys(mapping)).join(','); +} + function setup( paramName, targetMap ) { return function( raw, clean ){ return sanitize( raw, clean, { paramName: paramName, targetMap: targetMap, - targetMapKeysString: Object.keys(targetMap).join(',') + targetMapKeysString: getValidKeys(targetMap) }); }; } function sanitize( raw, clean, opts ) { - // error & warning messages var messages = { errors: [], warnings: [] }; - // init clean.types - clean.types = clean.types || {}; - // the string of targets (comma delimeted) var targetsString = raw[opts.paramName]; @@ -51,19 +50,13 @@ function sanitize( raw, clean, opts ) { // only set types value when no error occured if( !messages.errors.length ){ - - // store the values under a new key as 'clean.types.from_*' - var typesKey = 'from_' + opts.paramName; - - // ? - clean.types[typesKey] = targets.reduce(function(acc, target) { + clean[opts.paramName] = targets.reduce(function(acc, target) { return acc.concat(opts.targetMap[target]); }, []); // dedupe in case aliases expanded to common things or user typed in duplicates - clean.types[typesKey] = _.uniq(clean.types[typesKey]); + clean[opts.paramName] = _.uniq(clean[opts.paramName]); } - } } diff --git a/sanitiser/_text.js b/sanitiser/_text.js index 53898ff0..e6897a5e 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -23,10 +23,6 @@ function sanitize( raw, clean ){ if (check.assigned(parsed_text)) { clean.parsed_text = parsed_text; } - - // try to set layers from query parser results - clean.types = clean.layers || {}; - clean.types.from_text_parser = text_parser.get_layers(clean.text); } return messages; diff --git a/sanitiser/reverse.js b/sanitiser/reverse.js index 154fd3d9..045ff3a1 100644 --- a/sanitiser/reverse.js +++ b/sanitiser/reverse.js @@ -3,8 +3,8 @@ var type_mapping = require('../helper/type_mapping'); var sanitizeAll = require('../sanitiser/sanitizeAll'), sanitizers = { singleScalarParameters: require('../sanitiser/_single_scalar_parameters'), - layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type), - sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type), + layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping), + sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), size: require('../sanitiser/_size'), private: require('../sanitiser/_flag_bool')('private', false), geo_reverse: require('../sanitiser/_geo_reverse'), diff --git a/sanitiser/search.js b/sanitiser/search.js index cc596b26..1be32b02 100644 --- a/sanitiser/search.js +++ b/sanitiser/search.js @@ -5,8 +5,8 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'), singleScalarParameters: require('../sanitiser/_single_scalar_parameters'), text: require('../sanitiser/_text'), size: require('../sanitiser/_size'), - layers: require('../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type), - sources: require('../sanitiser/_targets')('sources', type_mapping.source_to_type), + layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping), + sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), private: require('../sanitiser/_flag_bool')('private', false), geo_search: require('../sanitiser/_geo_search'), boundary_country: require('../sanitiser/_boundary_country'), diff --git a/service/mget.js b/service/mget.js index a878e8cb..529d0fc1 100644 --- a/service/mget.js +++ b/service/mget.js @@ -24,7 +24,6 @@ function service( backend, query, cb ){ // query new backend backend().client.mget( cmd, function( err, data ){ - // handle backend errors if( err ){ return cb( err ); } diff --git a/test/unit/controller/place.js b/test/unit/controller/place.js index e8a8c333..977be59e 100644 --- a/test/unit/controller/place.js +++ b/test/unit/controller/place.js @@ -41,7 +41,7 @@ module.exports.tests.functional_success = function(test, common) { test('functional success', function(t) { var backend = mockBackend( 'client/mget/ok/1', function( cmd ){ - t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command'); + t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command'); }); var controller = setup( backend ); var res = { @@ -57,7 +57,7 @@ module.exports.tests.functional_success = function(test, common) { t.deepEqual(json.features, expected, 'values correctly mapped'); } }; - var req = { clean: { ids: [ {'id' : 123, types: [ 'a' ] } ] }, errors: [], warnings: [] }; + var req = { clean: { ids: [ {'id' : 123, layers: [ 'a' ] } ] }, errors: [], warnings: [] }; var next = function next() { t.equal(req.errors.length, 0, 'next was called without error'); t.end(); @@ -70,10 +70,10 @@ module.exports.tests.functional_success = function(test, common) { module.exports.tests.functional_failure = function(test, common) { test('functional failure', function(t) { var backend = mockBackend( 'client/mget/fail/1', function( cmd ){ - t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command'); + t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: ['b'] } ] } }, 'correct backend command'); }); var controller = setup( backend ); - var req = { clean: { ids: [ {'id' : 123, types: [ 'b' ] } ] }, errors: [], warnings: [] }; + var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] }; var next = function( message ){ t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler'); t.end(); diff --git a/test/unit/helper/text_parser.js b/test/unit/helper/text_parser.js index 3accf7b9..90b8fa52 100644 --- a/test/unit/helper/text_parser.js +++ b/test/unit/helper/text_parser.js @@ -1,7 +1,7 @@ var parser = require('../../../helper/text_parser'); var type_mapping = require('../../../helper/type_mapping'); -var layers_map = type_mapping.layer_with_aliases_to_type; +var layers_map = type_mapping.layer_mapping; module.exports.tests = {}; diff --git a/test/unit/helper/type_mapping.js b/test/unit/helper/type_mapping.js index b44b00a9..39d9c564 100644 --- a/test/unit/helper/type_mapping.js +++ b/test/unit/helper/type_mapping.js @@ -4,132 +4,32 @@ var type_mapping = require('../../../helper/type_mapping'); module.exports.tests = {}; module.exports.tests.interfaces = function(test, common) { - test('types list', function(t) { - t.ok(check.array(type_mapping.types), 'is array'); - t.ok(check.hasLength(type_mapping.types, 11), 'has correct number of elements'); + test('basic layer mapping', function(t) { + t.deepEquals(type_mapping.layer_mapping.venue, ['venue']); + t.deepEquals(type_mapping.layer_mapping.address, ['address']); t.end(); }); - test('type to source mapping', function(t) { - t.ok(check.object(type_mapping.type_to_source), 'is object'); - t.ok(check.hasLength(Object.keys(type_mapping.type_to_source), 11), 'has correct number of elements'); + test('alias layer mapping', function(t) { + t.deepEquals(type_mapping.layer_mapping.coarse, + [ 'continent', 'macrocountry', 'country', 'dependency', + 'region', 'locality', 'localadmin', 'county', 'macrohood', + 'neighbourhood', 'microhood', 'disputed', 'admin0', + 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin']); t.end(); }); - test('type to layer mapping', function(t) { - t.ok(check.object(type_mapping.type_to_layer), 'is object'); - t.ok(check.hasLength(Object.keys(type_mapping.type_to_layer), 11), 'has correct number of elements'); + test('basic source mapping', function(t) { + t.deepEquals(type_mapping.source_mapping.osm, ['openstreetmap']); + t.deepEquals(type_mapping.source_mapping.openaddresses, ['openaddresses']); t.end(); }); - test('source to type mapping', function(t) { - t.ok(check.object(type_mapping.source_to_type), 'is object'); - t.ok(check.hasLength(Object.keys(type_mapping.source_to_type), 8), 'has correct number of elements'); + test('alias source mapping', function(t) { + t.deepEquals(type_mapping.source_mapping.openstreetmap, ['openstreetmap']); + t.deepEquals(type_mapping.source_mapping.wof, ['whosonfirst']); t.end(); }); - - test('layer to type mapping', function(t) { - t.ok(check.object(type_mapping.layer_to_type), 'is object'); - t.equal(Object.keys(type_mapping.layer_to_type).length, 8, 'has correct number of elements'); - t.end(); - }); - - test('layer to type mapping (with aliases)', function(t) { - t.ok(check.object(type_mapping.layer_with_aliases_to_type), 'is object'); - t.ok(check.hasLength(Object.keys(type_mapping.layer_with_aliases_to_type), 9), 'has correct number of elements'); - t.end(); - }); - - test('\'osm\' and \'openstreetmap\' sources should only successfully map to \'venue\' and \'address\' layers', function(t) { - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'venue'), ['osmnode', 'osmway']); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'address'), ['osmaddress']); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('osm', 'coarse'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'venue'), ['osmnode', 'osmway']); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'address'), ['osmaddress']); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openstreetmap', 'coarse'), []); - t.end(); - }); - - test('\'gn\' and \'geonames\' sources should only successfully map to \'venue\' layers', function(t) { - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'venue'), ['geoname']); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'address'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('gn', 'coarse'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'venue'), ['geoname']); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'address'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('geonames', 'coarse'), []); - t.end(); - }); - - test('\'oa\' and \'openaddresses\' sources should only successfully map to \'address\' layer', function(t) { - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'venue'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'address'), ['openaddresses']); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('oa', 'coarse'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'venue'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'address'), ['openaddresses']); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'country'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'region'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'county'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'locality'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'localadmin'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'neighbourhood'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('openaddresses', 'coarse'), []); - t.end(); - }); - - test('\'qs\' and \'quattroshapes\' sources should only successfully map to \'address\' layer', function(t) { - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'venue'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'address'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'country'), ['admin0']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'region'), ['admin1']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'county'), ['admin2']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'locality'), ['locality']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'localadmin'), ['local_admin']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'neighbourhood'), ['neighborhood']); - t.deepEquals(type_mapping.source_and_layer_to_type('qs', 'coarse'), - ['admin0','admin1','admin2','neighborhood','locality','local_admin']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'venue'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'address'), []); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'country'), ['admin0']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'region'), ['admin1']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'county'), ['admin2']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'locality'), ['locality']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'localadmin'), ['local_admin']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'neighbourhood'), ['neighborhood']); - t.deepEquals(type_mapping.source_and_layer_to_type('quattroshapes', 'coarse'), - ['admin0','admin1','admin2','neighborhood','locality','local_admin']); - t.end(); - }); - }; module.exports.all = function (tape, common) { diff --git a/test/unit/helper/types.js b/test/unit/helper/types.js deleted file mode 100644 index 4c68b46c..00000000 --- a/test/unit/helper/types.js +++ /dev/null @@ -1,92 +0,0 @@ -var types = require('../../../helper/types'); - -module.exports.tests = {}; - -module.exports.tests.no_cleaned_types = function(test, common) { - test('no cleaned types', function(t) { - function testIt() { - types({}); - } - - t.throws(testIt, /clean_types should not be null or undefined/, 'no input should result in exception'); - - t.end(); - }); -}; - -module.exports.tests.address_parser = function(test, common) { - test('address parser specifies only admin layers', function(t) { - var cleaned_types = { - from_text_parser: ['admin0'] // simplified return value from address parser - }; - var actual = types(cleaned_types); - var expected = ['admin0']; // simplified expected value for all admin layers - t.deepEqual(actual, expected, 'only layers specified by address parser returned'); - t.end(); - }); -}; - -module.exports.tests.layers_parameter = function(test, common) { - test('layers parameter specifies only some layers', function(t) { - var cleaned_types = { - from_layers: ['geoname'] - }; - var actual = types(cleaned_types); - var expected = ['geoname']; - t.deepEqual(actual, expected, 'only types specified by layers parameter returned'); - t.end(); - }); -}; - -module.exports.tests.layers_parameter_and_address_parser = function(test, common) { - test('layers parameter and address parser present', function(t) { - var cleaned_types = { - from_layers: ['geoname'], - from_text_parser: ['admin0'] // simplified return value from address parse - }; - var actual = types(cleaned_types); - var expected = ['geoname']; - t.deepEqual(actual, expected, 'layers parameter overrides address parser completely'); - t.end(); - }); -}; - -module.exports.tests.source_parameter = function(test, common) { - test('source parameter specified', function(t) { - var cleaned_types = { - from_sources: ['openaddresses'] - }; - - var actual = types(cleaned_types); - - var expected = ['openaddresses']; - t.deepEqual(actual, expected, 'type parameter set to types specified by source'); - t.end(); - }); -}; - -module.exports.tests.source_and_layers_parameters = function(test, common) { - test('source and layers parameter both specified', function(t) { - var cleaned_types = { - from_sources: ['openaddresses'], - from_layers: ['osmaddress', 'openaddresses'] - }; - - var actual = types(cleaned_types); - - var expected = ['openaddresses']; - t.deepEqual(actual, expected, 'type set to intersection of source and layer types'); - t.end(); - }); -}; - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('types: ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; diff --git a/test/unit/run.js b/test/unit/run.js index c600e0ad..89b7c31b 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -15,7 +15,6 @@ var tests = [ require('./helper/labelSchema'), require('./helper/text_parser'), require('./helper/type_mapping'), - require('./helper/types'), require('./helper/sizeCalculator'), require('./middleware/confidenceScore'), require('./middleware/confidenceScoreReverse'), diff --git a/test/unit/sanitiser/_ids.js b/test/unit/sanitiser/_ids.js index 98237180..64e749b0 100644 --- a/test/unit/sanitiser/_ids.js +++ b/test/unit/sanitiser/_ids.js @@ -94,60 +94,79 @@ module.exports.tests.valid_ids = function(test, common) { test('ids: valid input (openaddresses)', function(t) { var raw = { ids: 'openaddresses:address:20' }; var clean = {}; - var expected_ids = [{ - id: '20', - types: [ 'openaddresses' ] - }]; var messages = sanitize( raw, clean ); + var expected_ids = [{ + source: 'openaddresses', + layer: 'address', + id: '20', + }]; t.deepEqual( messages.errors, [], ' no errors'); t.deepEqual( clean.ids, expected_ids, 'single type value returned'); t.end(); }); test('ids: valid input (osm)', function(t) { - var raw = { ids: 'osm:venue:500' }; + var raw = { ids: 'openstreetmap:venue:node:500' }; var clean = {}; var expected_ids = [{ - id: '500', - types: [ 'osmnode', 'osmway' ] + source: 'openstreetmap', + layer: 'venue', + id: 'node:500', }]; var messages = sanitize( raw, clean ); t.deepEqual( messages.errors, [], ' no errors'); - t.deepEqual( clean.ids, expected_ids, 'osm could be two types, but that\'s ok'); + t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field'); t.end(); }); }; module.exports.tests.multiple_ids = function(test, common) { test('multiple ids', function(t) { - var raw = { ids: 'geonames:venue:1,osm:venue:2' }; + var raw = { ids: 'geonames:venue:1,openstreetmap:address:way:2' }; var clean = {}; - var expected_clean = { ids: [ { id: '1', types: [ 'geoname' ] }, { id: '2', types: [ 'osmnode', 'osmway' ] } ] }; var messages = sanitize( raw, clean); + var expected_ids = [ { + source: 'geonames', + layer: 'venue', + id: '1' + }, { + source: 'openstreetmap', + layer: 'address', + id: 'way:2' + } ]; + t.deepEqual( messages.errors, [], 'no errors' ); t.deepEqual( messages.warnings, [], 'no warnings' ); - t.deepEqual(clean, expected_clean, 'clean set correctly'); + t.deepEqual(clean.ids, expected_ids, 'clean set correctly'); t.end(); }); }; module.exports.tests.de_dupe = function(test, common) { test('duplicate ids', function(t) { - var expected_clean = { ids: [ { id: '1', types: [ 'geoname' ] }, { id: '2', types: [ 'osmnode', 'osmway' ] } ] }; - var raw = { ids: 'geonames:venue:1,osm:venue:2,geonames:venue:1' }; + var raw = { ids: 'geonames:venue:1,openstreetmap:venue:node:2,geonames:venue:1' }; var clean = {}; var messages = sanitize( raw, clean ); + var expected_ids = [ { + source: 'geonames', + layer: 'venue', + id: '1' + }, { + source: 'openstreetmap', + layer: 'venue', + id: 'node:2' + } ]; t.deepEqual( messages.errors, [], 'no errors' ); t.deepEqual( messages.warnings, [], 'no warnings' ); - t.deepEqual(clean, expected_clean, 'clean set correctly'); + t.deepEqual(clean.ids, expected_ids, 'clean set correctly'); t.end(); }); }; diff --git a/test/unit/sanitiser/_layers.js b/test/unit/sanitiser/_layers.js index 16ca19ae..bd499fc7 100644 --- a/test/unit/sanitiser/_layers.js +++ b/test/unit/sanitiser/_layers.js @@ -1,6 +1,6 @@ var type_mapping = require('../../../helper/type_mapping'); -var sanitize = require('../../../sanitiser/_targets')('layers', type_mapping.layer_with_aliases_to_type); +var sanitize = require('../../../sanitiser/_targets')('layers', type_mapping.layer_mapping); module.exports.tests = {}; @@ -25,95 +25,101 @@ module.exports.tests.sanitize_layers = function(test, common) { }); test('venue (alias) layer', function(t) { - var venue_layers = ['geoname','osmnode','osmway']; var raw = { layers: 'venue' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers set'); + var venue_layers = ['venue']; + t.deepEqual(clean.layers, venue_layers, 'venue layers set'); t.end(); }); test('coarse (alias) layer', function(t) { - var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin']; var raw = { layers: 'coarse' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, admin_layers, 'coarse layers set'); + var admin_layers = [ 'continent', 'macrocountry', 'country', 'dependency', + 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', + 'microhood', 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin' ]; + + t.deepEqual(clean.layers, admin_layers, 'coarse layers set'); t.end(); }); - test('address (alias) layer', function(t) { - var address_layers = ['osmaddress','openaddresses']; + test('address layer', function(t) { var raw = { layers: 'address' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, address_layers, 'address layers set'); + t.deepEqual(clean.layers, ['address'], 'address layer set'); t.end(); }); test('venue alias layer plus regular layers', function(t) { - var venue_layers = ['geoname','osmnode','osmway']; - var reg_layers = ['admin0', 'admin1']; var raw = { layers: 'venue,country,region' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, venue_layers.concat(reg_layers), 'venue + regular layers'); + var expected_layers = ['venue', 'country', 'admin0', 'region', 'admin1']; + t.deepEqual(clean.layers, expected_layers, 'venue + regular layers'); t.end(); }); test('coarse alias layer plus regular layers', function(t) { - var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin']; - var reg_layers = ['geoname', 'osmnode', 'osmway']; - - var raw = { layers: 'coarse,venue,country' }; + var raw = { layers: 'coarse,country' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, admin_layers.concat(reg_layers), 'coarse + regular layers set'); + var expected_layers = [ 'continent', 'macrocountry', 'country', 'dependency', + 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', + 'microhood', 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin' ]; + + t.deepEqual(clean.layers, expected_layers, 'coarse + regular layers set'); t.end(); }); test('address alias layer plus regular layers', function(t) { - var address_layers = ['osmaddress','openaddresses']; - var reg_layers = ['admin0', 'locality']; - var raw = { layers: 'address,country,locality' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, address_layers.concat(reg_layers), 'address + regular layers set'); + var expected_layers = ['address', 'country', 'admin0', 'locality' ]; + t.deepEqual(clean.layers, expected_layers, 'address + regular layers set'); t.end(); }); test('alias layer plus regular layers (no duplicates)', function(t) { - var venue_layers = ['geoname','osmnode','osmway','admin0']; var raw = { layers: 'venue,country' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers found (no duplicates)'); + var expected_layers = ['venue', 'country', 'admin0']; + t.deepEqual(clean.layers, expected_layers, 'venue layers found (no duplicates)'); t.end(); }); test('multiple alias layers (no duplicates)', function(t) { - var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin']; var raw = { layers: 'venue,coarse' }; var clean = {}; sanitize(raw, clean); - t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)'); + var coarse_layers = [ 'continent', 'macrocountry', + 'country', 'dependency', 'region', 'locality', 'localadmin', + 'county', 'macrohood', 'neighbourhood', 'microhood', + 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin']; + + var venue_layers = [ 'venue' ]; + var expected_layers = venue_layers.concat(coarse_layers); + t.deepEqual(clean.layers, expected_layers, 'all layers found (no duplicates)'); t.end(); }); }; diff --git a/test/unit/sanitiser/_sources.js b/test/unit/sanitiser/_sources.js index d2ef405f..61b2be90 100644 --- a/test/unit/sanitiser/_sources.js +++ b/test/unit/sanitiser/_sources.js @@ -1,5 +1,5 @@ var type_mapping = require('../../../helper/type_mapping'); -var sanitize = require( '../../../sanitiser/_targets' )('sources', type_mapping.source_to_type); +var sanitize = require( '../../../sanitiser/_targets' )('sources', type_mapping.source_mapping); var success_messages = { error: false }; @@ -14,7 +14,7 @@ module.exports.tests.no_sources = function(test, common) { var messages = sanitize(req.query, req.clean); - t.deepEqual(req.clean.types, {}, 'clean.types should be empty object'); + t.equal(req.clean.sources, undefined, 'no sources should be defined'); t.deepEqual(messages.errors, [], 'no error returned'); t.deepEqual(messages.warnings, [], 'no warnings returned'); t.end(); @@ -29,11 +29,11 @@ module.exports.tests.no_sources = function(test, common) { }; var expected_error = 'sources parameter cannot be an empty string. ' + - 'Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap'; + 'Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst'; var messages = sanitize(req.query, req.clean); - t.deepEqual(req.clean.types, {}, 'clean.types should be empty object'); + t.equal(req.clean.sources, undefined, 'no sources should be defined'); t.deepEqual(messages.errors.length, 1, 'error returned'); t.deepEqual(messages.errors[0], expected_error, 'error returned'); t.deepEqual(messages.warnings, [], 'no warnings returned'); @@ -52,27 +52,24 @@ module.exports.tests.valid_sources = function(test, common) { var messages = sanitize(req.query, req.clean); - t.deepEqual(req.clean.types, { from_sources: ['geoname'] }, 'clean.types should contain from_source entry with geonames'); + t.deepEqual(req.clean.sources, ['geonames'], 'sources should contain geonames'); t.deepEqual(messages.errors, [], 'no error returned'); t.deepEqual(messages.warnings, [], 'no warnings returned'); t.end(); }); - test('openstreetmap source', function(t) { + test('openstreetmap (abbreviated) source', function(t) { var req = { query: { - sources: 'openstreetmap' + sources: 'osm' }, clean: { } }; - var expected_types = { - from_sources: ['osmaddress', 'osmnode', 'osmway'] - }; var messages = sanitize(req.query, req.clean); - t.deepEqual(req.clean.types, expected_types, 'clean.types should contain from_source entry with multiple entries for openstreetmap'); + t.deepEqual(req.clean.sources, ['openstreetmap'], 'abbreviation is expanded to full version'); t.deepEqual(messages.errors, [], 'no error returned'); t.deepEqual(messages.warnings, [], 'no warnings returned'); t.end(); @@ -85,14 +82,11 @@ module.exports.tests.valid_sources = function(test, common) { }, clean: { } }; - var expected_types = { - from_sources: ['osmaddress', 'osmnode', 'osmway', 'openaddresses'] - }; var messages = sanitize(req.query, req.clean); - t.deepEqual(req.clean.types, expected_types, - 'clean.types should contain from_source entry with multiple entries for openstreetmap and openadresses'); + t.deepEqual(req.clean.sources, ['openstreetmap', 'openaddresses'], + 'clean.sources should contain openstreetmap and openadresses'); t.deepEqual(messages.errors, [], 'no error returned'); t.deepEqual(messages.warnings, [], 'no warnings returned'); t.end(); @@ -109,7 +103,8 @@ module.exports.tests.invalid_sources = function(test, common) { }; var expected_messages = { errors: [ - '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' + '\'notasource\' is an invalid sources parameter. ' + + 'Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' ], warnings: [] }; @@ -117,7 +112,7 @@ module.exports.tests.invalid_sources = function(test, common) { var messages = sanitize(req.query, req.clean); t.deepEqual(messages, expected_messages, 'error with message returned'); - t.deepEqual(req.clean.types, { }, 'clean.types should remain empty'); + t.equal(req.clean.sources, undefined, 'clean.sources should remain empty'); t.end(); }); }; diff --git a/test/unit/sanitiser/_text.js b/test/unit/sanitiser/_text.js index 8bc0710f..3868c86c 100644 --- a/test/unit/sanitiser/_text.js +++ b/test/unit/sanitiser/_text.js @@ -15,7 +15,6 @@ module.exports.tests.text_parser = function(test, common) { t.deepEquals(messages.errors, [], 'no errors'); t.deepEquals(messages.warnings, [], 'no warnings'); - t.equal(clean.types.from_text_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred'); t.end(); }); diff --git a/test/unit/sanitiser/place.js b/test/unit/sanitiser/place.js index f7cda1e9..dd9d1645 100644 --- a/test/unit/sanitiser/place.js +++ b/test/unit/sanitiser/place.js @@ -1,7 +1,7 @@ var place = require('../../../sanitiser/place'), sanitize = place.sanitize, middleware = place.middleware, - defaultClean = { ids: [ { id: '123', types: [ 'geoname' ] } ], private: false }; + defaultClean = { ids: [ { source: 'geonames', layer: 'venue', id: '123' } ], private: false }; // these are the default values you would expect when no input params are specified. module.exports.tests = {}; diff --git a/test/unit/sanitiser/reverse.js b/test/unit/sanitiser/reverse.js index 4e727b5b..a55568e8 100644 --- a/test/unit/sanitiser/reverse.js +++ b/test/unit/sanitiser/reverse.js @@ -11,15 +11,13 @@ var reverse = require('../../../sanitiser/reverse'), 'boundary.circle.lat': 0, 'boundary.circle.lon': 0, 'boundary.circle.radius': parseFloat(defaults['boundary:circle:radius']), - types: { - }, size: 10, private: false }; // these are the default values you would expect when no input params are specified. // @todo: why is this different from $defaultClean? -var emptyClean = { private: false, size: 10, types: {} }; +var emptyClean = { private: false, size: 10 }; module.exports.tests = {}; @@ -38,7 +36,8 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.sanitisers = function(test, common) { test('check sanitiser list', function (t) { - var expected = ['singleScalarParameters', 'layers', 'sources', 'size', 'private', 'geo_reverse', 'boundary_country']; + var expected = ['singleScalarParameters', 'layers', 'sources', + 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country']; t.deepEqual(Object.keys(reverse.sanitiser_list), expected); t.end(); }); @@ -104,7 +103,7 @@ module.exports.tests.sanitize_lon = function(test, common) { var req = { query: { 'point.lat': 0, 'point.lon': lon } }; // @todo: why is lat set? - var expected = { 'point.lat': 0, private: false, size: 10, types: {} }; + var expected = { 'point.lat': 0, private: false, size: 10 }; sanitize(req, function(){ t.equal(req.errors[0], 'missing param \'point.lon\'', 'longitude is a required field'); t.deepEqual(req.clean, expected, 'clean only has default values set'); diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index b00160dd..d42d87f4 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -5,7 +5,7 @@ var extend = require('extend'), middleware = search.middleware, defaultError = 'invalid param \'text\': text length, must be >0'; // these are the default values you would expect when no input params are specified. -var emptyClean = { private: false, size: 10, types: {} }; +var emptyClean = { private: false, size: 10 }; module.exports.tests = {}; @@ -24,7 +24,8 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.sanitisers = function(test, common) { test('check sanitiser list', function (t) { - var expected = ['singleScalarParameters', 'text', 'size', 'layers', 'sources', 'private', 'geo_search', 'boundary_country' ]; + var expected = ['singleScalarParameters', 'text', 'size', 'layers', 'sources', + 'sources_and_layers', 'private', 'geo_search', 'boundary_country' ]; t.deepEqual(Object.keys(search.sanitiser_list), expected); t.end(); }); From ce292baeafbdacf11c12c271995183dc60da610a Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 24 Feb 2016 12:37:54 +0100 Subject: [PATCH 040/106] fixes for focus.point to work with wof integration --- query/view/focus_selected_layers.js | 6 ++---- .../fixture/autocomplete_linguistic_focus.js | 18 ++++-------------- ...utocomplete_linguistic_focus_null_island.js | 18 ++++-------------- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/query/view/focus_selected_layers.js b/query/view/focus_selected_layers.js index 038d7ffa..ba19919f 100644 --- a/query/view/focus_selected_layers.js +++ b/query/view/focus_selected_layers.js @@ -22,10 +22,8 @@ module.exports = function( subview ){ if( view && view.hasOwnProperty('function_score') ){ view.function_score.filter = { 'or': [ - { 'type': { 'value': 'osmnode' } }, - { 'type': { 'value': 'osmway' } }, - { 'type': { 'value': 'osmaddress' } }, - { 'type': { 'value': 'openaddresses' } } + { 'term': { 'layer': 'venue' } }, + { 'term': { 'layer': 'address' } } ] }; } diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 7f3c7174..83156c0b 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -47,23 +47,13 @@ module.exports = { 'filter': { 'or': [ { - 'type': { - 'value': 'osmnode' + 'term': { + 'layer': 'venue' } }, { - 'type': { - 'value': 'osmway' - } - }, - { - 'type': { - 'value': 'osmaddress' - } - }, - { - 'type': { - 'value': 'openaddresses' + 'term': { + 'layer': 'address' } } ] diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 46554715..f28b6234 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -47,23 +47,13 @@ module.exports = { 'filter': { 'or': [ { - 'type': { - 'value': 'osmnode' + 'term': { + 'layer': 'venue' } }, { - 'type': { - 'value': 'osmway' - } - }, - { - 'type': { - 'value': 'osmaddress' - } - }, - { - 'type': { - 'value': 'openaddresses' + 'term': { + 'layer': 'address' } } ] From 616dd606a731ae5e57bd7cf0cacc999b9d820d33 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 15:44:00 -0500 Subject: [PATCH 041/106] Add sources and layers sanitiser This sanitiser can do a better job of determining when an invalid combination of sources and layers was specified, and produce helpful error messages. --- sanitiser/_sources_and_layers.js | 37 +++++++ sanitiser/reverse.js | 2 + sanitiser/search.js | 2 + test/unit/run.js | 1 + test/unit/sanitiser/_sources_and_layers.js | 115 +++++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 sanitiser/_sources_and_layers.js create mode 100644 test/unit/sanitiser/_sources_and_layers.js diff --git a/sanitiser/_sources_and_layers.js b/sanitiser/_sources_and_layers.js new file mode 100644 index 00000000..448a7af1 --- /dev/null +++ b/sanitiser/_sources_and_layers.js @@ -0,0 +1,37 @@ +var _ = require( 'lodash' ); +var type_mapping = require( '../helper/type_mapping' ); + +/* + * This sanitiser depends on clean.layers and clean.sources + * so it has to be run after those sanitisers have been run + */ +function sanitize( raw, clean ){ + var messages = { errors: [], warnings: [] }; + + var possible_errors = []; + var at_least_one_valid_combination = false; + + if (clean.layers && clean.sources) { + clean.sources.forEach(function(source) { + var layers_for_source = type_mapping.layers_by_source[source]; + clean.layers.forEach(function(layer) { + if (_.includes(layers_for_source, layer)) { + at_least_one_valid_combination = true; + } else { + var message = 'You have specified both the `sources` and `layers` ' + + 'parameters in a combination that will return no results: the ' + + source + ' source has nothing in the ' + layer + ' layer'; + possible_errors.push(message); + } + }); + }); + + if (!at_least_one_valid_combination) { + messages.errors = possible_errors; + } + } + + return messages; +} + +module.exports = sanitize; diff --git a/sanitiser/reverse.js b/sanitiser/reverse.js index 045ff3a1..d707fcea 100644 --- a/sanitiser/reverse.js +++ b/sanitiser/reverse.js @@ -5,6 +5,8 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'), singleScalarParameters: require('../sanitiser/_single_scalar_parameters'), layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), + // depends on the layers and sources sanitisers, must be run after them + sources_and_layers: require('../sanitiser/_sources_and_layers'), size: require('../sanitiser/_size'), private: require('../sanitiser/_flag_bool')('private', false), geo_reverse: require('../sanitiser/_geo_reverse'), diff --git a/sanitiser/search.js b/sanitiser/search.js index 1be32b02..980b5a04 100644 --- a/sanitiser/search.js +++ b/sanitiser/search.js @@ -7,6 +7,8 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'), size: require('../sanitiser/_size'), layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), + // depends on the layers and sources sanitisers, must be run after them + sources_and_layers: require('../sanitiser/_sources_and_layers'), private: require('../sanitiser/_flag_bool')('private', false), geo_search: require('../sanitiser/_geo_search'), boundary_country: require('../sanitiser/_boundary_country'), diff --git a/test/unit/run.js b/test/unit/run.js index 89b7c31b..9d210853 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -37,6 +37,7 @@ var tests = [ require('./sanitiser/_single_scalar_parameters'), require('./sanitiser/_size'), require('./sanitiser/_sources'), + require('./sanitiser/_sources_and_layers'), require('./sanitiser/_text'), require('./sanitiser/autocomplete'), require('./sanitiser/place'), diff --git a/test/unit/sanitiser/_sources_and_layers.js b/test/unit/sanitiser/_sources_and_layers.js new file mode 100644 index 00000000..96001f4e --- /dev/null +++ b/test/unit/sanitiser/_sources_and_layers.js @@ -0,0 +1,115 @@ +var sanitize = require('../../../sanitiser/_sources_and_layers'); + +var type_mapping = require('../../../helper/type_mapping'); + +module.exports.tests = {}; + +module.exports.tests.inactive = function(test, common) { + test('no source or layer specified', function(t) { + var raw = {}; + var clean = {}; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); + + test('only layers specified', function(t) { + var raw = {}; + var clean = { layers: ['venue'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); + + test('only sources specified', function(t) { + var raw = {}; + var clean = { sources: ['openstreetmap'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); +}; + +module.exports.tests.no_errors = function(test, common) { + test('valid combination', function(t) { + var raw = {}; + var clean = { sources: ['openstreetmap'], layers: ['venue'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); + + test('valid combination because of multiple sources', function(t) { + var raw = {}; + var clean = { sources: ['openstreetmap', 'openaddresses'], layers: ['venue'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); + + test('valid combination because of multiple layers', function(t) { + var raw = {}; + var clean = { sources: ['openaddresses'], layers: ['address', 'country'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 0, 'should return no errors'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); +}; + +module.exports.tests.invalid_combination = function(test, common) { + test('address layer with wof', function(t) { + var raw = {}; + var clean = { sources: ['whosonfirst'], layers: ['address'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 1, 'should return an error'); + t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' + + 'parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); + + test('admin layers with osm', function(t) { + var raw = {}; + var clean = { sources: ['openstreetmap'], layers: ['country', 'locality'] }; + + var messages = sanitize(raw, clean); + + t.equal(messages.errors.length, 2, 'should return an error'); + t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' + + 'parameters in a combination that will return no results: the openstreetmap source has nothing in the country layer'); + t.equal(messages.errors[1], 'You have specified both the `sources` and `layers` ' + + 'parameters in a combination that will return no results: the openstreetmap source has nothing in the locality layer'); + t.equal(messages.warnings.length, 0, 'should return no warnings'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANTIZE _sources_and_layers ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; From 67449dcae074501d66fd24f00babb25b07a6b7ba Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 24 Feb 2016 12:37:54 +0100 Subject: [PATCH 042/106] Tweak weights for focus --- query/autocomplete_defaults.js | 6 +++--- test/unit/fixture/autocomplete_linguistic_final_token.js | 2 +- test/unit/fixture/autocomplete_linguistic_focus.js | 6 +++--- .../fixture/autocomplete_linguistic_focus_null_island.js | 6 +++--- .../unit/fixture/autocomplete_linguistic_multiple_tokens.js | 2 +- test/unit/fixture/autocomplete_linguistic_only.js | 2 +- test/unit/fixture/autocomplete_linguistic_with_admin.js | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 634f5165..df1b206f 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -30,10 +30,10 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'phrase:slop': 2, 'focus:function': 'linear', - 'focus:offset': '10km', + 'focus:offset': '0km', 'focus:scale': '250km', 'focus:decay': 0.5, - 'focus:weight': 3, + 'focus:weight': 10, 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'multiply', @@ -90,6 +90,6 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'population:field': 'population', 'population:modifier': 'log1p', 'population:max_boost': 20, - 'population:weight': 2 + 'population:weight': 3 }); diff --git a/test/unit/fixture/autocomplete_linguistic_final_token.js b/test/unit/fixture/autocomplete_linguistic_final_token.js index e63c84b6..1641ac44 100644 --- a/test/unit/fixture/autocomplete_linguistic_final_token.js +++ b/test/unit/fixture/autocomplete_linguistic_final_token.js @@ -70,7 +70,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 }] } }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 83156c0b..513136b9 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -35,12 +35,12 @@ module.exports = { 'lat': 29.49136, 'lon': -82.50622 }, - 'offset': '10km', + 'offset': '0km', 'scale': '250km', 'decay': 0.5 } }, - 'weight': 3 + 'weight': 10 }], 'score_mode': 'avg', 'boost_mode': 'multiply', @@ -114,7 +114,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 }] } }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index f28b6234..3985d061 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -35,12 +35,12 @@ module.exports = { 'lat': 0, 'lon': 0 }, - 'offset': '10km', + 'offset': '0km', 'scale': '250km', 'decay': 0.5 } }, - 'weight': 3 + 'weight': 10 }], 'score_mode': 'avg', 'boost_mode': 'multiply', @@ -114,7 +114,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 }] } }] diff --git a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js index dab5e31a..16a25605 100644 --- a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js +++ b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js @@ -81,7 +81,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 }] } }] diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 2ec025d4..05ff6c43 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -70,7 +70,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 }] } }] diff --git a/test/unit/fixture/autocomplete_linguistic_with_admin.js b/test/unit/fixture/autocomplete_linguistic_with_admin.js index 215a5fda..2174fcbf 100644 --- a/test/unit/fixture/autocomplete_linguistic_with_admin.js +++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js @@ -133,7 +133,7 @@ module.exports = { 'modifier': 'log1p', 'field': 'population' }, - 'weight': 2 + 'weight': 3 } ], 'score_mode': 'first', From ab9dbdf7f070642960296e46c86be7cc26cc9cf0 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 24 Feb 2016 18:50:00 -0500 Subject: [PATCH 043/106] Update ciao tests --- test/ciao/place/basic_place.coffee | 2 +- test/ciao/reverse/layers_alias_coarse.coffee | 20 +++++++++++++++++-- test/ciao/reverse/layers_invalid.coffee | 2 +- .../reverse/layers_mix_invalid_valid.coffee | 2 +- test/ciao/reverse/layers_multiple.coffee | 3 +-- test/ciao/reverse/layers_single.coffee | 3 +-- test/ciao/reverse/sources_invalid.coffee | 2 +- .../sources_layers_invalid_combo.coffee | 7 +++---- .../reverse/sources_layers_valid_combo.coffee | 4 ++-- test/ciao/reverse/sources_multiple.coffee | 3 +-- test/ciao/reverse/sources_single.coffee | 3 +-- test/ciao/search/layers_alias_address.coffee | 3 +-- test/ciao/search/layers_alias_coarse.coffee | 20 +++++++++++++++++-- test/ciao/search/layers_invalid.coffee | 2 +- .../search/layers_mix_invalid_valid.coffee | 3 ++- test/ciao/search/layers_multiple.coffee | 3 +-- test/ciao/search/layers_single.coffee | 3 +-- test/ciao/search/sources_invalid.coffee | 2 +- .../sources_layers_invalid_combo.coffee | 6 +++--- .../search/sources_layers_valid_combo.coffee | 4 ++-- test/ciao/search/sources_multiple.coffee | 3 +-- test/ciao/search/sources_single.coffee | 3 +-- 22 files changed, 63 insertions(+), 40 deletions(-) diff --git a/test/ciao/place/basic_place.coffee b/test/ciao/place/basic_place.coffee index b4a18711..e46bbee8 100644 --- a/test/ciao/place/basic_place.coffee +++ b/test/ciao/place/basic_place.coffee @@ -29,5 +29,5 @@ should.not.exist json.geocoding.errors should.not.exist json.geocoding.warnings #? inputs -json.geocoding.query['ids'].should.eql [{ id: '1', types: [ 'geoname' ] }] +json.geocoding.query['ids'].should.eql [{ id: '1', layer: 'venue', source: 'geonames' }] should.not.exist json.geocoding.query['size'] diff --git a/test/ciao/reverse/layers_alias_coarse.coffee b/test/ciao/reverse/layers_alias_coarse.coffee index 13671a05..6b911083 100644 --- a/test/ciao/reverse/layers_alias_coarse.coffee +++ b/test/ciao/reverse/layers_alias_coarse.coffee @@ -30,5 +30,21 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] -json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] \ No newline at end of file +json.geocoding.query.layers.should.eql [ "continent", + "macrocountry", + "country", + "dependency", + "region", + "locality", + "localadmin", + "county", + "macrohood", + "neighbourhood", + "microhood", + "disputed", + "admin0", + "admin1", + "admin2", + "neighborhood", + "local_admin" +] diff --git a/test/ciao/reverse/layers_invalid.coffee b/test/ciao/reverse/layers_invalid.coffee index 354efb02..a7105815 100644 --- a/test/ciao/reverse/layers_invalid.coffee +++ b/test/ciao/reverse/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_mix_invalid_valid.coffee b/test/ciao/reverse/layers_mix_invalid_valid.coffee index a636a26b..f8e07adf 100644 --- a/test/ciao/reverse/layers_mix_invalid_valid.coffee +++ b/test/ciao/reverse/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_multiple.coffee b/test/ciao/reverse/layers_multiple.coffee index f5f2d7dd..d85c07a2 100644 --- a/test/ciao/reverse/layers_multiple.coffee +++ b/test/ciao/reverse/layers_multiple.coffee @@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"] -json.geocoding.query['type'].should.eql ["admin0","admin1"] +json.geocoding.query.layers.should.eql ["country","admin0","region","admin1"] diff --git a/test/ciao/reverse/layers_single.coffee b/test/ciao/reverse/layers_single.coffee index 4b65c331..ab1f8e08 100644 --- a/test/ciao/reverse/layers_single.coffee +++ b/test/ciao/reverse/layers_single.coffee @@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0"] -json.geocoding.query['type'].should.eql ["admin0"] +json.geocoding.query.layers.should.eql ["country","admin0"] diff --git a/test/ciao/reverse/sources_invalid.coffee b/test/ciao/reverse/sources_invalid.coffee index bdc6c826..f0b54803 100644 --- a/test/ciao/reverse/sources_invalid.coffee +++ b/test/ciao/reverse/sources_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ] +json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/sources_layers_invalid_combo.coffee b/test/ciao/reverse/sources_layers_invalid_combo.coffee index 17de70ca..984830a7 100644 --- a/test/ciao/reverse/sources_layers_invalid_combo.coffee +++ b/test/ciao/reverse/sources_layers_invalid_combo.coffee @@ -24,13 +24,12 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ] +json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the quattroshapes source has nothing in the address layer' ] #? expected warnings should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"] -json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] -should.not.exist json.geocoding.query['type'] +json.geocoding.query.layers.should.eql ["address"] +json.geocoding.query.sources.should.eql ["quattroshapes"] diff --git a/test/ciao/reverse/sources_layers_valid_combo.coffee b/test/ciao/reverse/sources_layers_valid_combo.coffee index a24ea221..fc2fee76 100644 --- a/test/ciao/reverse/sources_layers_valid_combo.coffee +++ b/test/ciao/reverse/sources_layers_valid_combo.coffee @@ -30,5 +30,5 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"] -json.geocoding.query['type'].should.eql ["openaddresses"] +json.geocoding.query.layers.should.eql ["address"] +json.geocoding.query.sources.should.eql ["openaddresses"] diff --git a/test/ciao/reverse/sources_multiple.coffee b/test/ciao/reverse/sources_multiple.coffee index 8728109c..4fd24366 100644 --- a/test/ciao/reverse/sources_multiple.coffee +++ b/test/ciao/reverse/sources_multiple.coffee @@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"] -json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"] +json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"] diff --git a/test/ciao/reverse/sources_single.coffee b/test/ciao/reverse/sources_single.coffee index 2c638503..6a062c9a 100644 --- a/test/ciao/reverse/sources_single.coffee +++ b/test/ciao/reverse/sources_single.coffee @@ -30,5 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"] -json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"] +json.geocoding.query.sources.should.eql ["openstreetmap"] diff --git a/test/ciao/search/layers_alias_address.coffee b/test/ciao/search/layers_alias_address.coffee index 9bf32a04..e3e39d15 100644 --- a/test/ciao/search/layers_alias_address.coffee +++ b/test/ciao/search/layers_alias_address.coffee @@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"] -json.geocoding.query['type'].should.eql ["osmaddress","openaddresses"] +json.geocoding.query.layers.should.eql ["address"] diff --git a/test/ciao/search/layers_alias_coarse.coffee b/test/ciao/search/layers_alias_coarse.coffee index 262be88e..c2527df0 100644 --- a/test/ciao/search/layers_alias_coarse.coffee +++ b/test/ciao/search/layers_alias_coarse.coffee @@ -31,5 +31,21 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] -json.geocoding.query['type'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] \ No newline at end of file +json.geocoding.query.layers.should.eql [ "continent", + "macrocountry", + "country", + "dependency", + "region", + "locality", + "localadmin", + "county", + "macrohood", + "neighbourhood", + "microhood", + "disputed", + "admin0", + "admin1", + "admin2", + "neighborhood", + "local_admin" +] diff --git a/test/ciao/search/layers_invalid.coffee b/test/ciao/search/layers_invalid.coffee index 1e8e65fc..40de3f85 100644 --- a/test/ciao/search/layers_invalid.coffee +++ b/test/ciao/search/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_mix_invalid_valid.coffee b/test/ciao/search/layers_mix_invalid_valid.coffee index 11a40ddc..6f1ccec6 100644 --- a/test/ciao/search/layers_mix_invalid_valid.coffee +++ b/test/ciao/search/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,venue,address,country,region,county,locality,localadmin,neighbourhood' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings @@ -32,3 +32,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 +should.not.exist json.geocoding.query['layers'] diff --git a/test/ciao/search/layers_multiple.coffee b/test/ciao/search/layers_multiple.coffee index 5f714d4d..c735a206 100644 --- a/test/ciao/search/layers_multiple.coffee +++ b/test/ciao/search/layers_multiple.coffee @@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0","admin1"] -json.geocoding.query['type'].should.eql ["admin0","admin1"] +json.geocoding.query.layers.should.eql ["country","admin0","region","admin1"] diff --git a/test/ciao/search/layers_single.coffee b/test/ciao/search/layers_single.coffee index 227f7bc9..ad104fab 100644 --- a/test/ciao/search/layers_single.coffee +++ b/test/ciao/search/layers_single.coffee @@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["admin0"] -json.geocoding.query['type'].should.eql ["admin0"] +json.geocoding.query.layers.should.eql ["country","admin0"] diff --git a/test/ciao/search/sources_invalid.coffee b/test/ciao/search/sources_invalid.coffee index 1d8df87c..8b6b9d27 100644 --- a/test/ciao/search/sources_invalid.coffee +++ b/test/ciao/search/sources_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap' ] +json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/sources_layers_invalid_combo.coffee b/test/ciao/search/sources_layers_invalid_combo.coffee index c0f84b13..d1e0fc07 100644 --- a/test/ciao/search/sources_layers_invalid_combo.coffee +++ b/test/ciao/search/sources_layers_invalid_combo.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results.' ] +json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the quattroshapes source has nothing in the address layer' ] #? expected warnings should.not.exist json.geocoding.warnings @@ -32,6 +32,6 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"] -json.geocoding.query.types['from_sources'].should.eql ["admin0","admin1","admin2","neighborhood","locality","local_admin"] +json.geocoding.query.layers.should.eql ["address"] +json.geocoding.query.sources.should.eql ["quattroshapes"] should.not.exist json.geocoding.query['type'] diff --git a/test/ciao/search/sources_layers_valid_combo.coffee b/test/ciao/search/sources_layers_valid_combo.coffee index eeb2a663..b17a81b0 100644 --- a/test/ciao/search/sources_layers_valid_combo.coffee +++ b/test/ciao/search/sources_layers_valid_combo.coffee @@ -31,5 +31,5 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_layers'].should.eql ["osmaddress","openaddresses"] -json.geocoding.query['type'].should.eql ["openaddresses"] +json.geocoding.query.layers.should.eql ["address"] +json.geocoding.query.sources.should.eql ["openaddresses"] diff --git a/test/ciao/search/sources_multiple.coffee b/test/ciao/search/sources_multiple.coffee index fa5a650c..08352113 100644 --- a/test/ciao/search/sources_multiple.coffee +++ b/test/ciao/search/sources_multiple.coffee @@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway","geoname"] -json.geocoding.query['type'].should.eql ["geoname","osmnode","osmway","osmaddress"] +json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"] diff --git a/test/ciao/search/sources_single.coffee b/test/ciao/search/sources_single.coffee index 3b3e0bf4..e639a8b5 100644 --- a/test/ciao/search/sources_single.coffee +++ b/test/ciao/search/sources_single.coffee @@ -31,5 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.types['from_sources'].should.eql ["osmaddress","osmnode","osmway"] -json.geocoding.query['type'].should.eql ["osmnode","osmway","osmaddress"] +json.geocoding.query.sources.should.eql ["openstreetmap"] From 74d05867cfee895d16a217e32b9dd371941178e7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 14:49:50 -0500 Subject: [PATCH 044/106] Update mock Elasticsearch response data --- test/unit/helper/geojsonify.js | 35 ++++++++++------------------------ 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index a5311050..3ab4d975 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -42,6 +42,7 @@ module.exports.tests.search = function(test, common) { '_id': 'id1', '_type': 'layer1', 'source': 'source1', + 'layer': 'layer1', 'center_point': { 'lat': 51.5337144, 'lon': -0.1069716 @@ -60,12 +61,6 @@ module.exports.tests.search = function(test, common) { 'localadmin': 'test1', 'locality': 'test2', 'neighbourhood': 'test3', - 'suggest': { - 'input': [ - '\'round midnight jazz and blues bar' - ], - 'output': 'osmnode:2208150035' - }, 'category': [ 'food', 'nightlife' @@ -75,6 +70,7 @@ module.exports.tests.search = function(test, common) { '_id': 'id2', '_type': 'layer2', 'source': 'source2', + 'layer': 'layer2', 'name': { 'default': 'Blues Cafe' }, @@ -90,17 +86,12 @@ module.exports.tests.search = function(test, common) { 'localadmin': 'test1', 'locality': 'test2', 'neighbourhood': 'test3', - 'suggest': { - 'input': [ - 'blues cafe' - ], - 'output': 'osmway:147495160' - } }, { - '_id': '34633854', - '_type': 'way', - 'source': 'osm', + '_id': 'node:34633854', + '_type': 'venue', + 'source': 'openstreetmap', + 'layer': 'venue', 'name': { 'default': 'Empire State Building' }, @@ -116,12 +107,6 @@ module.exports.tests.search = function(test, common) { 'localadmin': 'Manhattan', 'locality': 'New York', 'neighbourhood': 'Koreatown', - 'suggest': { - 'input': [ - 'empire state building' - ], - 'output': 'osmway:34633854' - }, 'category': [ 'tourism', 'transport' @@ -198,10 +183,10 @@ module.exports.tests.search = function(test, common) { ] }, 'properties': { - 'id': '34633854', - 'gid': 'osm:way:34633854', - 'layer': 'way', - 'source': 'osm', + 'id': 'node:34633854', + 'gid': 'openstreetmap:venue:node:34633854', + 'layer': 'venue', + 'source': 'openstreetmap', 'label': 'Empire State Building, Manhattan, NY, USA', 'name': 'Empire State Building', 'country_a': 'USA', From 692b6da17dcea409cf9cd2edc69bf5bb7736fa28 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 14:50:49 -0500 Subject: [PATCH 045/106] Use layer instead of _type field to determine layer --- helper/geojsonify.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index b415b18f..8ed808f5 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -28,12 +28,8 @@ function lookupSource(src) { return src.source; } -/* - * Use the type to layer mapping, except for Geonames, where having a full - * Elasticsearch document source allows a more specific layer name to be chosen - */ function lookupLayer(src) { - return src._type; + return src.layer; } function geojsonifyPlaces( docs ){ From 79883594b8611391fba187997d9e2069c38860a6 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 14:55:43 -0500 Subject: [PATCH 046/106] Use clean.layers to determine types This allows removing the types middleware and simplifies some code. --- controller/search.js | 10 +++------- middleware/_types.js | 18 ------------------ routes/v1.js | 4 ---- 3 files changed, 3 insertions(+), 29 deletions(-) delete mode 100644 middleware/_types.js diff --git a/controller/search.js b/controller/search.js index 4adaf31d..a564e7b4 100644 --- a/controller/search.js +++ b/controller/search.js @@ -11,7 +11,6 @@ function setup( backend, query ){ // do not run controller when a request // validation error has occurred. if( req.errors && req.errors.length ){ - delete req.clean.type; //to declutter output return next(); } @@ -25,12 +24,9 @@ function setup( backend, query ){ body: query( req.clean ) }; - // set the Elasticsearch types to filter by, - // and remove the property from clean so the API - // response output is cleaner - if( req.clean.hasOwnProperty('type') ){ - cmd.type = req.clean.type; - delete req.clean.type; + // use layers field for filtering by type + if( req.clean.hasOwnProperty('layers') ){ + cmd.type = req.clean.layers; } logger.debug( '[ES req]', JSON.stringify(cmd) ); diff --git a/middleware/_types.js b/middleware/_types.js deleted file mode 100644 index ccabdab9..00000000 --- a/middleware/_types.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Take the layers specified by the layers parameter and use them to set the - * list of Elasticsearch types to filter. - * - * This has to be done outside the layers sanitizer since it doesn't know that - * the layers property is eventualy used to choose the _type. - */ -function middleware(req, res, next) { - req.clean = req.clean || {}; - - if (req.clean.hasOwnProperty('layers')) { - req.clean.type = req.clean.layers; - } - - next(); -} - -module.exports = middleware; diff --git a/routes/v1.js b/routes/v1.js index 49c9ea4c..00c99aa5 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -12,7 +12,6 @@ var sanitisers = { /** ----------------------- middleware ------------------------ **/ var middleware = { - types: require('../middleware/_types'), calcSize: require('../middleware/sizeCalculator') }; @@ -59,7 +58,6 @@ function addRoutes(app, peliasConfig) { ]), search: createRouter([ sanitisers.search.middleware, - middleware.types, middleware.calcSize(), controllers.search(), postProc.distances('focus.point.'), @@ -72,7 +70,6 @@ function addRoutes(app, peliasConfig) { ]), autocomplete: createRouter([ sanitisers.autocomplete.middleware, - middleware.types, controllers.search(null, require('../query/autocomplete')), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), @@ -84,7 +81,6 @@ function addRoutes(app, peliasConfig) { ]), reverse: createRouter([ sanitisers.reverse.middleware, - middleware.types, middleware.calcSize(), controllers.search(undefined, reverseQuery), postProc.distances('point.'), From e14697fe406b1453ea2e4670f17ca6f4b7e113d8 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 11:20:51 -0500 Subject: [PATCH 047/106] Remove adminFields because it is not needed anymore --- helper/adminFields.js | 45 ------------------ test/unit/helper/adminFields.js | 84 --------------------------------- 2 files changed, 129 deletions(-) delete mode 100644 helper/adminFields.js delete mode 100644 test/unit/helper/adminFields.js diff --git a/helper/adminFields.js b/helper/adminFields.js deleted file mode 100644 index 31648bc4..00000000 --- a/helper/adminFields.js +++ /dev/null @@ -1,45 +0,0 @@ - -var _ = require('lodash'), - peliasSchema = require('pelias-schema'), - peliasLogger = require( 'pelias-logger' ).get( 'api' ); - -var ADMIN_FIELDS = [ - 'admin0', - 'admin1', - 'admin1_abbr', - 'admin2', - 'local_admin', - 'locality', - 'neighborhood', - 'address.zip' -]; - -/** - * Get all admin fields that were expected and also found in schema - * - * @param {Object} [schema] optional: for testing only - * @param {Array} [expectedFields] optional: for testing only - * @param {Object} [logger] optional: for testing only - * @returns {Array.} - */ -function getAvailableAdminFields(schema, expectedFields, logger) { - - schema = schema || peliasSchema; - expectedFields = expectedFields || ADMIN_FIELDS; - logger = logger || peliasLogger; - - var actualFields = Object.keys(schema.mappings._default_.properties); - - // check if expected fields are actually in current schema - var available = expectedFields.filter(function (field) { - return _.includes( actualFields, field ); - }); - - if (available.length === 0) { - logger.error('helper/adminFields: no expected admin fields found in schema'); - } - - return available; -} - -module.exports = getAvailableAdminFields; diff --git a/test/unit/helper/adminFields.js b/test/unit/helper/adminFields.js deleted file mode 100644 index 8780b267..00000000 --- a/test/unit/helper/adminFields.js +++ /dev/null @@ -1,84 +0,0 @@ -var adminFields = require('../../../helper/adminFields'); - -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('validate fields', function(t) { - t.assert(adminFields instanceof Function, 'adminFields is a function'); - t.assert(adminFields() instanceof Array, 'adminFields() returns an array'); - t.assert(adminFields().length > 0, 'adminFields array is not empty'); - t.end(); - }); -}; - -module.exports.tests.lookupExistance = function(test, common) { - test('all expected fields in schema', function(t) { - - var expectedFields = [ - 'one', - 'two', - 'three', - 'four' - ]; - var schema = { mappings: { _default_: { properties: {} } } }; - - // inject all expected fields into schema mock - expectedFields.forEach(function (field) { - schema.mappings._default_.properties[field] = {}; - }); - - var res = adminFields(schema, expectedFields); - - t.deepEquals(res, expectedFields, 'all expected fields are returned'); - t.end(); - }); - - test('some expected fields in schema', function(t) { - - var expectedFields = [ - 'one', - 'two', - 'three', - 'four' - ]; - var schema = { mappings: { _default_: { properties: {} } } }; - - // inject only some of the expected fields into schema mock - expectedFields.slice(0, 3).forEach(function (field) { - schema.mappings._default_.properties[field] = {}; - }); - - var res = adminFields(schema, expectedFields); - - t.deepEquals(res, expectedFields.slice(0, 3), 'only matching expected fields are returned'); - t.end(); - }); - - test('no expected fields in schema', function(t) { - - var schema = { mappings: { _default_: { properties: { foo: {} } } } }; - - var logErrorCalled = false; - var logger = { - error: function () { - logErrorCalled = true; - }}; - - var res = adminFields(schema, undefined, logger); - - t.deepEquals(res, [], 'no admin fields found'); - t.assert(logErrorCalled, 'log error called'); - t.end(); - }); -}; - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('adminFields: ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; \ No newline at end of file From 4219cefd56ce804473023f79092a367405d45e25 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 11:21:05 -0500 Subject: [PATCH 048/106] Add new parent properties and take the first one in each property if it is an array --- helper/geojsonify.js | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 8ed808f5..28fa6272 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -11,16 +11,25 @@ var DETAILS_PROPS = [ 'housenumber', 'street', 'postalcode', - 'country_a', + 'confidence', + 'distance', 'country', + 'country_id', + 'country_a', 'region', + 'region_id', 'region_a', 'county', + 'county_id', + 'county_a', 'localadmin', + 'localadmin_id', + 'localadmin_a', 'locality', + 'locality_id', + 'locality_a', 'neighbourhood', - 'confidence', - 'distance' + 'neighbourhood_id' ]; @@ -93,7 +102,7 @@ function addDetails(src, dst) { * @param {object} dst */ function addLabel(src, dst) { - dst.label = labelGenerator(src); + dst.label = labelGenerator(dst); } /** @@ -126,8 +135,23 @@ function computeBBox(geojson) { */ function copyProperties( source, props, dst ) { props.forEach( function ( prop ) { + if ( source.hasOwnProperty( prop ) ) { - dst[prop] = source[prop]; + + // array value, take first item in array (at this time only used for admin values) + if (source[prop] instanceof Array) { + if (source[prop].length === 0) { + return; + } + if (source[prop][0]) { + dst[prop] = source[prop][0]; + } + } + + // simple value + else { + dst[prop] = source[prop]; + } } }); } From b8ab7c00f79a6221c154bd886a73b89d8248656c Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 12:48:25 -0500 Subject: [PATCH 049/106] Cleanup the label generation code just a bit --- helper/labelGenerator.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index e772ceb9..e76c2bba 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -1,4 +1,3 @@ - var _ = require('lodash'), schemas = require('./labelSchema'); @@ -11,7 +10,6 @@ module.exports = function( record ){ var valueFunction = schema[key]; labelParts = valueFunction(record, labelParts); - } // NOTE: while it may seem odd to call `uniq` on the list of label parts, From 7aeea99ea719dc660c6ae7cf59bf88ce35dcae85 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 12:49:46 -0500 Subject: [PATCH 050/106] Remove quattroshapes and admin0/admin1 from layers --- helper/type_mapping.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/helper/type_mapping.js b/helper/type_mapping.js index ced5f1f3..f2e49d8f 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -49,7 +49,6 @@ var LAYERS_BY_SOURCE = { openstreetmap: [ 'address', 'venue' ], openaddresses: [ 'address' ], geonames: [ 'country', 'region', 'county', 'locality', 'venue' ], - quattroshapes: ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed'] }; @@ -60,9 +59,7 @@ var LAYERS_BY_SOURCE = { * may have layers that mean the same thing but have a different name */ var LAYER_ALIASES = { - 'coarse': LAYERS_BY_SOURCE.whosonfirst.concat(LAYERS_BY_SOURCE.quattroshapes), - 'country': ['country', 'admin0'], // Include both QS and WOF layers for various types of places - 'region': ['region', 'admin1'] // Alias layers that include themselves look weird, but do work + 'coarse': LAYERS_BY_SOURCE.whosonfirst }; // create a list of all layers by combining each entry from LAYERS_BY_SOURCE From 83de24d3c4ed2ba47688bbd57f5977d861e64407 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 12:52:36 -0500 Subject: [PATCH 051/106] Middleware: remove all use of alpha3/admin0/admin1 --- middleware/confidenceScore.js | 14 +++--- middleware/dedupe.js | 9 +++- middleware/localNamingConventions.js | 2 +- middleware/renamePlacenames.js | 69 ++++++++++++++++------------ 4 files changed, 55 insertions(+), 39 deletions(-) diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index 0eae0f8d..3cd9d1f3 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -84,7 +84,7 @@ function computeConfidenceScore(req, mean, stdev, hit) { /* * Check for clearly mismatching properties in a result - * zip code and state (admin1) are currently checked if present + * zip code and state (region) are currently checked if present * * @param {object|undefined} text * @param {object} hit @@ -95,8 +95,8 @@ function checkForDealBreakers(req, hit) { return false; } - if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.admin1_abbr) { - logger.debug('[confidence][deal-breaker]: state !== admin1_abbr'); + if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.parent.region_a) { + logger.debug('[confidence][deal-breaker]: state !== region_a'); return true; } @@ -210,8 +210,8 @@ function propMatch(textProp, hitProp, expectEnriched) { * @param {string|number} [hit.address.number] * @param {string} [hit.address.street] * @param {string|number} [hit.address.zip] - * @param {string} [hit.admin1_abbr] - * @param {string} [hit.alpha3] + * @param {string} [hit.parent.region_a] + * @param {string} [hit.parent.country_a] * @returns {number} */ function checkAddress(text, hit) { @@ -222,8 +222,8 @@ function checkAddress(text, hit) { res += propMatch(text.number, (hit.address ? hit.address.number : null), false); res += propMatch(text.street, (hit.address ? hit.address.street : null), false); res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true); - res += propMatch(text.state, hit.admin1_abbr, true); - res += propMatch(text.country, hit.alpha3, true); + res += propMatch(text.state, hit.parent.region_a, true); + res += propMatch(text.country, hit.parent.country_a, true); res /= checkCount; } diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 87bf20db..d0cce700 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -39,8 +39,13 @@ function dedupeResults(req, res, next) { */ function isDifferent(item1, item2) { try { - propMatch(item1, item2, 'admin1_abbr'); - propMatch(item1, item2, 'alpha3'); + if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) { + propMatch(item1.parent, item2.parent, 'region_a'); + propMatch(item1.parent, item2.parent, 'country'); + } + else if (item1.parent !== item2.parent) { + throw new Error('different'); + } if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) { propMatch(item1.name, item2.name, 'default'); diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index b7c5b8c9..084cfc26 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -13,7 +13,7 @@ function applyLocalNamingConventions(req, res, next) { // loop through data items and flip relevant number/street res.data.filter(function(place){ // only relevant for German addresses - if( 'DEU' !== place.alpha3 ){ return false; } + if( 'DEU' !== place.parent.country_a ){ return false; } if( !place.hasOwnProperty('address') ){ return false; } if( !place.address.hasOwnProperty('number') ){ return false; } if( !place.address.hasOwnProperty('street') ){ return false; } diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 6fcba91a..f3e35829 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -1,4 +1,4 @@ -var extend = require('extend'); +var _ = require('lodash'); /** - P is a preferred English name @@ -11,19 +11,34 @@ var extend = require('extend'); - A is an abbreviation or code for the place (e.g. "NYC" for New York) */ -// config mapping of old names to new ones -var NAME_MAP = { + +var ADDRESS_PROPS = { 'number': 'housenumber', 'zip': 'postalcode', - 'alpha3': 'country_a', - 'admin0': 'country', - 'admin1': 'region', - 'admin1_abbr': 'region_a', - 'admin2': 'county', - 'local_admin': 'localadmin', - 'neighborhood': 'neighbourhood' + 'street': 'street' }; +var PARENT_PROPS = [ + 'country', + 'country_id', + 'country_a', + 'region', + 'region_id', + 'region_a', + 'county', + 'county_id', + 'county_a', + 'localadmin', + 'localadmin_id', + 'localadmin_a', + 'locality', + 'locality_id', + 'locality_a', + 'neighbourhood', + 'neighbourhood_id' +]; + + function setup() { return renamePlacenames; @@ -36,31 +51,27 @@ function renamePlacenames(req, res, next) { return next(); } - // loop through data items and remap placenames - res.data = res.data.map(renameProperties); - + res.data = res.data.map(renameStuff); + next(); } -function renameProperties(place) { - var newPlace = {}; - Object.keys(place).forEach(function (property) { - if (property === 'address') { - extend(newPlace, renameProperties(place[property])); - } - else { - renameProperty(place, newPlace, property); - } - }); - return newPlace; -} +function renameStuff(place) { + + if (place.address) { + Object.keys(ADDRESS_PROPS).forEach(function (prop) { + place[ADDRESS_PROPS[prop]] = place.address[prop]; + }); + } -function renameProperty(oldObj, newObj, property) { - if (!oldObj.hasOwnProperty(property)) { - return; + // merge the parent block into the top level object to flatten the structure + if (place.parent) { + PARENT_PROPS.forEach(function (prop) { + place[prop] = place.parent[prop]; + }) } - newObj[(NAME_MAP[property] || property)] = oldObj[property]; + return place; } module.exports = setup; From 31374b5381d37d830222d54c92d818152c7327d8 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 12:54:43 -0500 Subject: [PATCH 052/106] Fix middleware unit tests --- test/unit/middleware/confidenceScore.js | 14 +++++++++--- .../unit/middleware/localNamingConventions.js | 22 +++++++++++-------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/test/unit/middleware/confidenceScore.js b/test/unit/middleware/confidenceScore.js index bf454381..7d516f16 100644 --- a/test/unit/middleware/confidenceScore.js +++ b/test/unit/middleware/confidenceScore.js @@ -70,13 +70,21 @@ module.exports.tests.confidenceScore = function(test, common) { value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - admin0: 'country1', admin1: 'state1', admin2: 'city1' + parent: { + country: 'country1', + region: 'state1', + county: 'city1' + } }, { + _score: 20, value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - admin0: 'country2', admin1: 'state2', admin2: 'city2', - _score: 20 + parent: { + country: 'country2', + region: 'state2', + county: 'city2' + } }], meta: {scores: [10]} }; diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index f3c877d0..da904e63 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -15,9 +15,11 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { 'number': '1', 'street': 'Main St' }, - 'admin1': 'Dungannon', - 'alpha3': 'GBR', - 'admin0': 'United Kingdom' + 'parent': { + 'region': 'Dungannon', + 'country_a': 'GBR', + 'country': 'United Kingdom' + } }; var deAddress = { @@ -30,12 +32,14 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { 'number': '23', 'street': 'Grolmanstraße' }, - 'admin1': 'Berlin', - 'locality': 'Berlin', - 'alpha3': 'DEU', - 'admin2': 'Berlin', - 'admin0': 'Germany', - 'neighborhood': 'Hansaviertel' + 'parent': { + 'region': 'Berlin', + 'locality': 'Berlin', + 'country_a': 'DEU', + 'county': 'Berlin', + 'country': 'Germany', + 'neighbourhood': 'Hansaviertel' + } }; var req = {}, From 3daec67b6b437b1dca53a7c6e2a6aab8c552992e Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 12:55:22 -0500 Subject: [PATCH 053/106] Fix helper unit tests --- test/unit/helper/labelGenerator_GBR.js | 6 ++-- test/unit/helper/labelGenerator_USA.js | 24 ++++++++-------- test/unit/helper/labelGenerator_default.js | 32 +++++++++++----------- test/unit/helper/type_mapping.js | 3 +- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/test/unit/helper/labelGenerator_GBR.js b/test/unit/helper/labelGenerator_GBR.js index 38ab3f91..a121444c 100644 --- a/test/unit/helper/labelGenerator_GBR.js +++ b/test/unit/helper/labelGenerator_GBR.js @@ -14,7 +14,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.one_main_street_uk = function(test, common) { test('one main street uk', function(t) { var doc = { - 'name': { 'default': '1 Main St' }, + 'name': '1 Main St', 'housenumber': '1', 'street': 'Main St', 'postalcode': 'BT77 0BG', @@ -31,7 +31,7 @@ module.exports.tests.one_main_street_uk = function(test, common) { module.exports.tests.hackney_city_farm = function(test, common) { test('hackney city farm', function(t) { var doc = { - 'name': { 'default': 'Hackney City Farm' }, + 'name': 'Hackney City Farm', 'country_a': 'GBR', 'country': 'United Kingdom', 'region': 'Hackney', @@ -48,7 +48,7 @@ module.exports.tests.hackney_city_farm = function(test, common) { module.exports.tests.wales = function(test, common) { test('wales', function(t) { var doc = { - 'name': { 'default': 'Wales' }, + 'name': 'Wales', 'country_a': 'GBR', 'country': 'United Kingdom', 'region': 'Wales' diff --git a/test/unit/helper/labelGenerator_USA.js b/test/unit/helper/labelGenerator_USA.js index e4b412ba..7791aa24 100644 --- a/test/unit/helper/labelGenerator_USA.js +++ b/test/unit/helper/labelGenerator_USA.js @@ -12,7 +12,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.localadmin = function(test, common) { test('localadmin should trump locality, neighbourhood, and county', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -30,7 +30,7 @@ module.exports.tests.localadmin = function(test, common) { module.exports.tests.locality = function(test, common) { test('locality should trump neighbourhood and county when localadmin not available', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -47,7 +47,7 @@ module.exports.tests.locality = function(test, common) { module.exports.tests.neighbourhood = function(test, common) { test('neighbourhood should trump county when neither localadmin nor locality', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -63,7 +63,7 @@ module.exports.tests.neighbourhood = function(test, common) { module.exports.tests.county = function(test, common) { test('county should be used when localadmin, locality, and neighbourhood are not available', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -78,7 +78,7 @@ module.exports.tests.county = function(test, common) { module.exports.tests.region = function(test, common) { test('region should be used when region_a is not available', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name' @@ -92,7 +92,7 @@ module.exports.tests.region = function(test, common) { module.exports.tests.region_geonames = function(test, common) { test('default name should not be prepended when source=geonames and layer=region', function(t) { var doc = { - 'name': { 'default': 'Region Name' }, + 'name': 'Region Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -109,7 +109,7 @@ module.exports.tests.region_geonames = function(test, common) { module.exports.tests.region_whosonfirst = function(test, common) { test('default name should not be prepended when source=whosonfirst and layer=region', function(t) { var doc = { - 'name': { 'default': 'Region Name' }, + 'name': 'Region Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -126,7 +126,7 @@ module.exports.tests.region_whosonfirst = function(test, common) { module.exports.tests.region_other_source = function(test, common) { test('default name should be prepended when layer=region and source is not whosonfirst or geonames', function(t) { var doc = { - 'name': { 'default': 'Default Name' }, + 'name': 'Default Name', 'country_a': 'USA', 'country': 'United States', 'region': 'Region Name', @@ -143,7 +143,7 @@ module.exports.tests.region_other_source = function(test, common) { module.exports.tests.san_francisco = function(test, common) { test('san francisco', function(t) { var doc = { - 'name': { 'default': 'San Francisco' }, + 'name': 'San Francisco', 'country_a': 'USA', 'country': 'United States', 'region': 'California', @@ -160,7 +160,7 @@ module.exports.tests.san_francisco = function(test, common) { module.exports.tests.nyc_office = function(test, common) { test('30 West 26th Street', function(t) { var doc = { - 'name': { 'default': '30 West 26th Street' }, + 'name': '30 West 26th Street', 'housenumber': '30', 'street': 'West 26th Street', 'postalcode': '10010', @@ -182,7 +182,7 @@ module.exports.tests.nyc_office = function(test, common) { module.exports.tests.nyc_bakery = function(test, common) { test('New York Bakery', function(t) { var doc = { - 'name': { 'default': 'New York Bakery' }, + 'name': 'New York Bakery', 'housenumber': '51 W', 'street': '29th', 'country_a': 'USA', @@ -203,7 +203,7 @@ module.exports.tests.nyc_bakery = function(test, common) { module.exports.tests.ferry_building = function(test, common) { test('Ferry Building', function(t) { var doc = { - 'name': { 'default': 'Ferry Building' }, + 'name': 'Ferry Building', 'country_a': 'USA', 'country': 'United States', 'region': 'California', diff --git a/test/unit/helper/labelGenerator_default.js b/test/unit/helper/labelGenerator_default.js index dc8ad02d..789233ba 100644 --- a/test/unit/helper/labelGenerator_default.js +++ b/test/unit/helper/labelGenerator_default.js @@ -14,7 +14,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.new_south_wales = function(test, common) { test('new south wales', function(t) { var doc = { - 'name': { 'default': 'New South Wales' }, + 'name': 'New South Wales', 'country_a': 'AUS', 'country': 'Australia', 'region': 'New South Wales' @@ -28,7 +28,7 @@ module.exports.tests.new_south_wales = function(test, common) { module.exports.tests.west_bengal = function(test, common) { test('west bengal', function(t) { var doc = { - 'name': { 'default': 'West Bengal' }, + 'name': 'West Bengal', 'country_a': 'IND', 'country': 'India', 'region': 'West Bengal' @@ -42,7 +42,7 @@ module.exports.tests.west_bengal = function(test, common) { module.exports.tests.bangalore = function(test, common) { test('bangalore', function(t) { var doc = { - 'name': { 'default': 'Bangalore' }, + 'name': 'Bangalore', 'country_a': 'IND', 'country': 'India', 'region': 'Karnataka', @@ -58,7 +58,7 @@ module.exports.tests.bangalore = function(test, common) { module.exports.tests.sarjapur = function(test, common) { test('Sarjapur', function(t) { var doc = { - 'name': { 'default': 'Sarjapur' }, + 'name': 'Sarjapur', 'country_a': 'IND', 'country': 'India', 'region': 'Karnataka' @@ -72,7 +72,7 @@ module.exports.tests.sarjapur = function(test, common) { module.exports.tests.bengaluru_east = function(test, common) { test('Bengaluru East', function(t) { var doc = { - 'name': { 'default': 'Bengaluru East' }, + 'name': 'Bengaluru East', 'country_a': 'IND', 'country': 'India', 'region': 'Karnataka', @@ -90,7 +90,7 @@ module.exports.tests.bengaluru_east = function(test, common) { module.exports.tests.wellington_victoria = function(test, common) { test('Wellington, Victoria, Australia', function(t) { var doc = { - 'name': { 'default': 'Wellington' }, + 'name': 'Wellington', 'country_a': 'AUS', 'country': 'Australia', 'region': 'Victoria', @@ -105,7 +105,7 @@ module.exports.tests.wellington_victoria = function(test, common) { module.exports.tests.arbil = function(test, common) { test('arbil', function(t) { var doc = { - 'name': { 'default': 'Arbil' }, + 'name': 'Arbil', 'country_a': 'IRQ', 'country': 'Iraq', 'region': 'Arbil' @@ -119,7 +119,7 @@ module.exports.tests.arbil = function(test, common) { module.exports.tests.madrid = function(test, common) { test('madrid', function(t) { var doc = { - 'name': { 'default': 'Madrid' }, + 'name': 'Madrid', 'country_a': 'ESP', 'country': 'Spain', 'region': 'Madrid' @@ -133,7 +133,7 @@ module.exports.tests.madrid = function(test, common) { module.exports.tests.one_grolmanstrasse = function(test, common) { test('one grolmanstrasse', function(t) { var doc = { - 'name': { 'default': '1 Grolmanstraße' }, + 'name': '1 Grolmanstraße', 'housenumber': '1', 'street': 'Grolmanstraße', 'postalcode': '10623', @@ -153,7 +153,7 @@ module.exports.tests.one_grolmanstrasse = function(test, common) { module.exports.tests.new_zealand = function(test, common) { test('new zealand', function(t) { var doc = { - 'name': { 'default': 'New Zealand' }, + 'name': 'New Zealand', 'country_a': 'NZL', 'country': 'New Zealand' }; @@ -166,7 +166,7 @@ module.exports.tests.new_zealand = function(test, common) { module.exports.tests.republic_of_ireland = function(test, common) { test('northern ireland', function(t) { var doc = { - 'name': { 'default': 'Ireland' }, + 'name': 'Ireland', 'country_a': 'IRL', 'country': 'Ireland' }; @@ -180,7 +180,7 @@ module.exports.tests.republic_of_ireland = function(test, common) { module.exports.tests.krabi_province = function(test, common) { test('Krabi Provence', function(t) { var doc = { - 'name': { 'default': 'Krabi' }, + 'name': 'Krabi', 'country_a': 'THA', 'country': 'Thailand', 'region': 'Krabi' @@ -194,7 +194,7 @@ module.exports.tests.krabi_province = function(test, common) { module.exports.tests.koh_lanta = function(test, common) { test('Koh Lanta', function(t) { var doc = { - 'name': { 'default': 'Ko Lanta' }, + 'name': 'Ko Lanta', 'country_a': 'THA', 'country': 'Thailand', 'region': 'Krabi' @@ -208,7 +208,7 @@ module.exports.tests.koh_lanta = function(test, common) { module.exports.tests.black_dog_cafe = function(test, common) { test('Black Dog Cafe', function(t) { var doc = { - 'name': { 'default': 'Black Dog Cafe' }, + 'name': 'Black Dog Cafe', 'country_a': 'NZL', 'country': 'New Zealand', 'region': 'Auckland Region', @@ -223,7 +223,7 @@ module.exports.tests.black_dog_cafe = function(test, common) { module.exports.tests.beach_bablyon = function(test, common) { test('Beach Bablyon', function(t) { var doc = { - 'name': { 'default': 'Beach Bablyon' }, + 'name': 'Beach Bablyon', 'country_a': 'NZL', 'country': 'New Zealand', 'region': 'Wellington Region', @@ -240,7 +240,7 @@ module.exports.tests.beach_bablyon = function(test, common) { module.exports.tests.waiotapu = function(test, common) { test('Waiotapu', function(t) { var doc = { - 'name': { 'default': 'Waiotapu' }, + 'name': 'Waiotapu', 'country_a': 'NZL', 'country': 'New Zealand', 'region': 'Bay of Plenty', diff --git a/test/unit/helper/type_mapping.js b/test/unit/helper/type_mapping.js index 39d9c564..2f075940 100644 --- a/test/unit/helper/type_mapping.js +++ b/test/unit/helper/type_mapping.js @@ -14,8 +14,7 @@ module.exports.tests.interfaces = function(test, common) { t.deepEquals(type_mapping.layer_mapping.coarse, [ 'continent', 'macrocountry', 'country', 'dependency', 'region', 'locality', 'localadmin', 'county', 'macrohood', - 'neighbourhood', 'microhood', 'disputed', 'admin0', - 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin']); + 'neighbourhood', 'microhood', 'disputed' ]); t.end(); }); From f3ba492e050368b0434972bc3b8795f9b18a2e0c Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 13:04:31 -0500 Subject: [PATCH 054/106] Remove all usage of alpha3/admin0/admin1, update query building to reflect new names in pelias-query --- query/autocomplete.js | 14 +++++------ query/autocomplete_defaults.js | 44 +++++++++++++++++----------------- query/reverse_defaults.js | 44 +++++++++++++++++----------------- query/search.js | 14 +++++------ query/search_defaults.js | 44 +++++++++++++++++----------------- query/text_parser.js | 27 +++++++++++++++------ 6 files changed, 100 insertions(+), 87 deletions(-) diff --git a/query/autocomplete.js b/query/autocomplete.js index bf5f76aa..4bc98c7c 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -27,14 +27,14 @@ query.score( peliasQuery.view.address('street') ); query.score( peliasQuery.view.address('postcode') ); // admin components -query.score( peliasQuery.view.admin('alpha3') ); -query.score( peliasQuery.view.admin('admin0') ); -query.score( peliasQuery.view.admin('admin1') ); -query.score( peliasQuery.view.admin('admin1_abbr') ); -query.score( peliasQuery.view.admin('admin2') ); -query.score( peliasQuery.view.admin('local_admin') ); +query.score( peliasQuery.view.admin('country') ); +query.score( peliasQuery.view.admin('country_a') ); +query.score( peliasQuery.view.admin('region') ); +query.score( peliasQuery.view.admin('region_a') ); +query.score( peliasQuery.view.admin('county') ); +query.score( peliasQuery.view.admin('localadmin') ); query.score( peliasQuery.view.admin('locality') ); -query.score( peliasQuery.view.admin('neighborhood') ); +query.score( peliasQuery.view.admin('neighbourhood') ); // scoring boost query.score( views.focus_selected_layers( views.ngrams_strict ) ); diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index df1b206f..7bb430cc 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -50,37 +50,37 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'address:postcode:field': 'address.zip', 'address:postcode:boost': 2000, - 'admin:alpha3:analyzer': 'standard', - 'admin:alpha3:field': 'alpha3', - 'admin:alpha3:boost': 1000, + 'admin:country_a:analyzer': 'standard', + 'admin:country_a:field': 'parent.country_a', + 'admin:country_a:boost': 1000, - 'admin:admin0:analyzer': 'peliasAdmin', - 'admin:admin0:field': 'admin0', - 'admin:admin0:boost': 800, + 'admin:country:analyzer': 'peliasAdmin', + 'admin:country:field': 'parent.country', + 'admin:country:boost': 800, - 'admin:admin1:analyzer': 'peliasAdmin', - 'admin:admin1:field': 'admin1', - 'admin:admin1:boost': 600, + 'admin:region:analyzer': 'peliasAdmin', + 'admin:region:field': 'parent.region', + 'admin:region:boost': 600, - 'admin:admin1_abbr:analyzer': 'peliasAdmin', - 'admin:admin1_abbr:field': 'admin1_abbr', - 'admin:admin1_abbr:boost': 600, + 'admin:region_a:analyzer': 'peliasAdmin', + 'admin:region_a:field': 'parent.region_a', + 'admin:region_a:boost': 600, - 'admin:admin2:analyzer': 'peliasAdmin', - 'admin:admin2:field': 'admin2', - 'admin:admin2:boost': 400, + 'admin:county:analyzer': 'peliasAdmin', + 'admin:county:field': 'parent.county', + 'admin:county:boost': 400, - 'admin:local_admin:analyzer': 'peliasAdmin', - 'admin:local_admin:field': 'local_admin', - 'admin:local_admin:boost': 200, + 'admin:localadmin:analyzer': 'peliasAdmin', + 'admin:localadmin:field': 'parent.localadmin', + 'admin:localadmin:boost': 200, 'admin:locality:analyzer': 'peliasAdmin', - 'admin:locality:field': 'locality', + 'admin:locality:field': 'parent.locality', 'admin:locality:boost': 200, - 'admin:neighborhood:analyzer': 'peliasAdmin', - 'admin:neighborhood:field': 'neighborhood', - 'admin:neighborhood:boost': 200, + 'admin:neighbourhood:analyzer': 'peliasAdmin', + 'admin:neighbourhood:field': 'parent.neighbourhood', + 'admin:neighbourhood:boost': 200, 'popularity:field': 'popularity', 'popularity:modifier': 'log1p', diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index 01e74773..c32b1c3d 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -50,37 +50,37 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'address:postcode:field': 'address.zip', 'address:postcode:boost': 3, - 'admin:alpha3:analyzer': 'standard', - 'admin:alpha3:field': 'alpha3', - 'admin:alpha3:boost': 5, + 'admin:country_a:analyzer': 'standard', + 'admin:country_a:field': 'parent.country_a', + 'admin:country_a:boost': 5, - 'admin:admin0:analyzer': 'peliasAdmin', - 'admin:admin0:field': 'admin0', - 'admin:admin0:boost': 4, + 'admin:country:analyzer': 'peliasAdmin', + 'admin:country:field': 'parent.country', + 'admin:country:boost': 4, - 'admin:admin1:analyzer': 'peliasAdmin', - 'admin:admin1:field': 'admin1', - 'admin:admin1:boost': 3, + 'admin:region:analyzer': 'peliasAdmin', + 'admin:region:field': 'parent.region', + 'admin:region:boost': 3, - 'admin:admin1_abbr:analyzer': 'peliasAdmin', - 'admin:admin1_abbr:field': 'admin1_abbr', - 'admin:admin1_abbr:boost': 3, + 'admin:region_a:analyzer': 'peliasAdmin', + 'admin:region_a:field': 'parent.region_a', + 'admin:region_a:boost': 3, - 'admin:admin2:analyzer': 'peliasAdmin', - 'admin:admin2:field': 'admin2', - 'admin:admin2:boost': 2, + 'admin:county:analyzer': 'peliasAdmin', + 'admin:county:field': 'parent.county', + 'admin:county:boost': 2, - 'admin:local_admin:analyzer': 'peliasAdmin', - 'admin:local_admin:field': 'local_admin', - 'admin:local_admin:boost': 1, + 'admin:localadmin:analyzer': 'peliasAdmin', + 'admin:localadmin:field': 'parent.localadmin', + 'admin:localadmin:boost': 1, 'admin:locality:analyzer': 'peliasAdmin', - 'admin:locality:field': 'locality', + 'admin:locality:field': 'parent.locality', 'admin:locality:boost': 1, - 'admin:neighborhood:analyzer': 'peliasAdmin', - 'admin:neighborhood:field': 'neighborhood', - 'admin:neighborhood:boost': 1, + 'admin:neighbourhood:analyzer': 'peliasAdmin', + 'admin:neighbourhood:field': 'parent.neighbourhood', + 'admin:neighbourhood:boost': 1, 'popularity:field': 'popularity', 'popularity:modifier': 'log1p', diff --git a/query/search.js b/query/search.js index 804cb88b..1db1a10a 100644 --- a/query/search.js +++ b/query/search.js @@ -25,14 +25,14 @@ query.score( peliasQuery.view.address('street') ); query.score( peliasQuery.view.address('postcode') ); // admin components -query.score( peliasQuery.view.admin('alpha3') ); -query.score( peliasQuery.view.admin('admin0') ); -query.score( peliasQuery.view.admin('admin1') ); -query.score( peliasQuery.view.admin('admin1_abbr') ); -query.score( peliasQuery.view.admin('admin2') ); -query.score( peliasQuery.view.admin('local_admin') ); +query.score( peliasQuery.view.admin('country') ); +query.score( peliasQuery.view.admin('country_a') ); +query.score( peliasQuery.view.admin('region') ); +query.score( peliasQuery.view.admin('region_a') ); +query.score( peliasQuery.view.admin('county') ); +query.score( peliasQuery.view.admin('localadmin') ); query.score( peliasQuery.view.admin('locality') ); -query.score( peliasQuery.view.admin('neighborhood') ); +query.score( peliasQuery.view.admin('neighbourhood') ); // non-scoring hard filters query.filter( peliasQuery.view.boundary_circle ); diff --git a/query/search_defaults.js b/query/search_defaults.js index 28cf6e7a..e9a918cd 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -50,37 +50,37 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'address:postcode:field': 'address.zip', 'address:postcode:boost': 20, - 'admin:alpha3:analyzer': 'standard', - 'admin:alpha3:field': 'alpha3', - 'admin:alpha3:boost': 5, + 'admin:country_a:analyzer': 'standard', + 'admin:country_a:field': 'parent.country_a', + 'admin:country_a:boost': 5, - 'admin:admin0:analyzer': 'peliasAdmin', - 'admin:admin0:field': 'admin0', - 'admin:admin0:boost': 4, + 'admin:country:analyzer': 'peliasAdmin', + 'admin:country:field': 'parent.country', + 'admin:country:boost': 4, - 'admin:admin1:analyzer': 'peliasAdmin', - 'admin:admin1:field': 'admin1', - 'admin:admin1:boost': 3, + 'admin:region:analyzer': 'peliasAdmin', + 'admin:region:field': 'parent.region', + 'admin:region:boost': 3, - 'admin:admin1_abbr:analyzer': 'peliasAdmin', - 'admin:admin1_abbr:field': 'admin1_abbr', - 'admin:admin1_abbr:boost': 3, + 'admin:region_a:analyzer': 'peliasAdmin', + 'admin:region_a:field': 'parent.region_a', + 'admin:region_a:boost': 3, - 'admin:admin2:analyzer': 'peliasAdmin', - 'admin:admin2:field': 'admin2', - 'admin:admin2:boost': 2, + 'admin:county:analyzer': 'peliasAdmin', + 'admin:county:field': 'parent.county', + 'admin:county:boost': 2, - 'admin:local_admin:analyzer': 'peliasAdmin', - 'admin:local_admin:field': 'local_admin', - 'admin:local_admin:boost': 1, + 'admin:localadmin:analyzer': 'peliasAdmin', + 'admin:localadmin:field': 'parent.localadmin', + 'admin:localadmin:boost': 1, 'admin:locality:analyzer': 'peliasAdmin', - 'admin:locality:field': 'locality', + 'admin:locality:field': 'parent.locality', 'admin:locality:boost': 1, - 'admin:neighborhood:analyzer': 'peliasAdmin', - 'admin:neighborhood:field': 'neighborhood', - 'admin:neighborhood:boost': 1, + 'admin:neighbourhood:analyzer': 'peliasAdmin', + 'admin:neighbourhood:field': 'parent.neighbourhood', + 'admin:neighbourhood:boost': 1, 'popularity:field': 'popularity', 'popularity:modifier': 'log1p', diff --git a/query/text_parser.js b/query/text_parser.js index 408d00f9..38fca48e 100644 --- a/query/text_parser.js +++ b/query/text_parser.js @@ -1,6 +1,20 @@ var logger = require('pelias-logger').get('api'); -var adminFields = require('../helper/adminFields')(); + +/* +This list should only contain admin fields we are comfortable matching in the case +when we can't identify parts of an address. This shouldn't contain fields like country_a +or postalcode because we should only try to match those when we're sure that's what they are. + */ +var adminFields = [ + 'country', + 'region', + 'region_a', + 'county', + 'localadmin', + 'locality', + 'neighbourhood' +]; /** @todo: refactor me @@ -48,17 +62,17 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ // city if( parsed_text.hasOwnProperty('city') ){ - vs.var( 'input:admin2', parsed_text.city ); + vs.var( 'input:county', parsed_text.city ); } // state if( parsed_text.hasOwnProperty('state') ){ - vs.var( 'input:admin1_abbr', parsed_text.state ); + vs.var( 'input:region_a', parsed_text.state ); } // country if( parsed_text.hasOwnProperty('country') ){ - vs.var( 'input:alpha3', parsed_text.country ); + vs.var( 'input:country_a', parsed_text.country ); } // ==== deal with the 'leftover' components ==== @@ -76,11 +90,10 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ // if we have 'leftovers' then assign them to any fields which // currently don't have a value assigned. if( leftoversString.length ){ - var unmatchedAdminFields = adminFields.slice(); - + // cycle through fields and set fields which // are still currently unset - unmatchedAdminFields.forEach( function( key ){ + adminFields.forEach( function( key ){ if( !vs.isset( 'input:' + key ) ){ vs.var( 'input:' + key, leftoversString ); } From af8044ef298de04d611d9b354ee9368b757d62ae Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 13:05:47 -0500 Subject: [PATCH 055/106] Fix all unit tests --- helper/labelGenerator.js | 2 +- test/unit/controller/search.js | 16 +- .../autocomplete_linguistic_with_admin.js | 14 +- .../dedupe_elasticsearch_nonascii_results.js | 38 ++-- .../fixture/dedupe_elasticsearch_results.js | 190 ++++++++++-------- .../fixture/reverse_with_boundary_country.js | 2 +- test/unit/fixture/search_boundary_country.js | 2 +- test/unit/fixture/search_full_address.js | 48 ++--- test/unit/fixture/search_partial_address.js | 38 ++-- test/unit/fixture/search_regions_address.js | 38 ++-- test/unit/helper/labelGenerator_SGP.js | 4 +- test/unit/helper/labelGenerator_SWE.js | 4 +- test/unit/mock/backend.js | 8 +- test/unit/query/search.js | 17 +- test/unit/sanitiser/_layers.js | 12 +- test/unit/sanitiser/search.js | 2 +- test/unit/service/mget.js | 4 +- test/unit/service/search.js | 4 +- 18 files changed, 240 insertions(+), 203 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index e76c2bba..7e9c1827 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -52,6 +52,6 @@ function getInitialLabel(record) { return []; } - return [record.name.default]; + return [record.name]; } diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js index 2b412013..3d46dde0 100644 --- a/test/unit/controller/search.js +++ b/test/unit/controller/search.js @@ -50,9 +50,11 @@ module.exports.tests.functional_success = function(test, common) { _id: 'myid1', _score: 10, _type: 'mytype1', - admin0: 'country1', - admin1: 'state1', - admin2: 'city1', + parent: { + country: 'country1', + region: 'state1', + county: 'city1' + }, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, value: 1 @@ -61,9 +63,11 @@ module.exports.tests.functional_success = function(test, common) { _id: 'myid2', _score: 20, _type: 'mytype2', - admin0: 'country2', - admin1: 'state2', - admin2: 'city2', + parent: { + country: 'country2', + region: 'state2', + county: 'city2' + }, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, value: 2 diff --git a/test/unit/fixture/autocomplete_linguistic_with_admin.js b/test/unit/fixture/autocomplete_linguistic_with_admin.js index 2174fcbf..2c27f670 100644 --- a/test/unit/fixture/autocomplete_linguistic_with_admin.js +++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js @@ -20,7 +20,7 @@ module.exports = { 'should': [ { 'match': { - 'admin0': { + 'parent.country': { 'analyzer': 'peliasAdmin', 'boost': 800, 'query': 'three' @@ -29,7 +29,7 @@ module.exports = { }, { 'match': { - 'admin1': { + 'parent.region': { 'analyzer': 'peliasAdmin', 'boost': 600, 'query': 'three' @@ -38,7 +38,7 @@ module.exports = { }, { 'match': { - 'admin1_abbr': { + 'parent.region_a': { 'analyzer': 'peliasAdmin', 'boost': 600, 'query': 'three' @@ -47,7 +47,7 @@ module.exports = { }, { 'match': { - 'admin2': { + 'parent.county': { 'analyzer': 'peliasAdmin', 'boost': 400, 'query': 'three' @@ -56,7 +56,7 @@ module.exports = { }, { 'match': { - 'local_admin': { + 'parent.localadmin': { 'analyzer': 'peliasAdmin', 'boost': 200, 'query': 'three' @@ -65,7 +65,7 @@ module.exports = { }, { 'match': { - 'locality': { + 'parent.locality': { 'analyzer': 'peliasAdmin', 'boost': 200, 'query': 'three' @@ -74,7 +74,7 @@ module.exports = { }, { 'match': { - 'neighborhood': { + 'parent.neighbourhood': { 'analyzer': 'peliasAdmin', 'boost': 200, 'query': 'three' diff --git a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js index 3341141a..ce929152 100644 --- a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js @@ -7,9 +7,11 @@ module.exports = [ 'name': { 'default': '万里长城万里长城' }, - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Beijing', + 'parent': { + 'country_a': 'CHN', + 'country': 'China', + 'region': 'Beijing' + }, 'confidence': 0.733 }, { @@ -20,9 +22,11 @@ module.exports = [ 'name': { 'default': '万里长城' }, - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Beijing', + 'parent': { + 'country_a': 'CHN', + 'country': 'China', + 'region': 'Beijing' + }, 'confidence': 0.733 }, { @@ -33,12 +37,14 @@ module.exports = [ 'name': { 'default': '万里花' }, - 'country_a': 'JPN', - 'country': 'Japan', - 'region': 'Tokyo', - 'county': '豊島区', - 'locality': 'Tokyo', - 'neighbourhood': '２丁目', + 'parent': { + 'country_a': 'JPN', + 'country': 'Japan', + 'region': 'Tokyo', + 'county': '豊島区', + 'locality': 'Tokyo', + 'neighbourhood': '２丁目' + }, 'confidence': 0.646 }, { @@ -53,9 +59,11 @@ module.exports = [ 'street': 'S308', 'postalcode': '312044' }, - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Zhejiang', + 'parent': { + 'country_a': 'CHN', + 'country': 'China', + 'region': 'Zhejiang' + }, 'confidence': 0.646 } ]; \ No newline at end of file diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index 6ed761c4..6f3d6d6c 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -5,17 +5,19 @@ module.exports = [ 'lat': 40.039265 }, 'address': {}, - 'local_admin': 'East Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'localadmin': 'East Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland' + }, 'name': { 'default': 'East Lampeter High School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Smoketown', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Greenland', 'category': [ 'education' ], @@ -30,17 +32,19 @@ module.exports = [ 'lat': 40.039265 }, 'address': {}, - 'local_admin': 'East Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'localadmin': 'East Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland' + }, 'name': { 'default': 'East Lampeter, High-School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Smoketown', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Greenland', 'category': [ 'education' ], @@ -55,17 +59,19 @@ module.exports = [ 'lat': 39.99288 }, 'address': {}, - 'local_admin': 'West Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'localadmin': 'West Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Lampeter', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Wheatland Mills' + }, 'name': { 'default': 'Lampeter-Strasburg High School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Lampeter', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Wheatland Mills', 'category': [ 'education' ], @@ -80,17 +86,19 @@ module.exports = [ 'lat': 40.03927 }, 'address': {}, - 'local_admin': 'East Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'localadmin': 'East Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland' + }, 'name': { 'default': 'East Lampeter High School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Smoketown', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Greenland', 'category': [ 'education' ], @@ -105,17 +113,19 @@ module.exports = [ 'lat': 39.992877 }, 'address': {}, - 'local_admin': 'West Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'region': 'Pennsylvania', + 'locality': 'Lampeter', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Wheatland Mills', + 'localadmin': 'West Lampeter', + 'region_a': 'PA' + }, 'name': { 'default': 'Lampeter-Strasburg High School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Lampeter', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Wheatland Mills', 'category': [ 'education' ], @@ -130,17 +140,19 @@ module.exports = [ 'lat': 40.038987 }, 'address': {}, - 'local_admin': 'East Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland', + 'localadmin': 'East Lampeter', + 'region_a': 'PA' + }, 'name': { 'default': 'East Lampeter School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Smoketown', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Greenland', 'category': [ 'education' ], @@ -155,17 +167,19 @@ module.exports = [ 'lat': 40.03899 }, 'address': {}, - 'local_admin': 'East Lampeter', - 'admin1_abbr': 'PA', + 'parent': { + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland', + 'localadmin': 'East Lampeter', + 'region_a': 'PA' + }, 'name': { 'default': 'East Lampeter School' }, - 'admin1': 'Pennsylvania', - 'locality': 'Smoketown', - 'alpha3': 'USA', - 'admin2': 'Lancaster County', - 'admin0': 'United States', - 'neighborhood': 'Greenland', 'category': [ 'education' ], @@ -180,16 +194,18 @@ module.exports = [ 'lat': 38.762788 }, 'address': {}, - 'local_admin': 'Polk', - 'admin1_abbr': 'MO', + 'parent': { + 'region': 'Missouri', + 'locality': 'Strasburg', + 'country_a': 'USA', + 'county': 'Cass County', + 'country': 'United States', + 'localadmin': 'Polk', + 'region_a': 'MO' + }, 'name': { 'default': 'Strasburg School' }, - 'admin1': 'Missouri', - 'locality': 'Strasburg', - 'alpha3': 'USA', - 'admin2': 'Cass County', - 'admin0': 'United States', 'category': [ 'education' ], @@ -204,16 +220,18 @@ module.exports = [ 'lat': 38.98445 }, 'address': {}, - 'admin1_abbr': 'VA', 'name': { 'default': 'Strasburg High School' }, - 'admin1': 'Virginia', - 'locality': 'Strasburg', - 'alpha3': 'USA', - 'admin2': 'Shenandoah County', - 'admin0': 'United States', - 'neighborhood': 'Strasburg Junction', + 'parent': { + 'region_a': 'VA', + 'region': 'Virginia', + 'locality': 'Strasburg', + 'country_a': 'USA', + 'county': 'Shenandoah County', + 'country': 'United States', + 'neighbourhood': 'Strasburg Junction' + }, 'category': [ 'education' ], @@ -228,16 +246,18 @@ module.exports = [ 'lat': 46.13427 }, 'address': {}, - 'local_admin': 'Strasburg', - 'admin1_abbr': 'ND', 'name': { 'default': 'Strasburg High School' }, - 'admin1': 'North Dakota', - 'locality': 'Strasburg', - 'alpha3': 'USA', - 'admin2': 'Emmons County', - 'admin0': 'United States', + 'parent': { + 'localadmin': 'Strasburg', + 'region_a': 'ND', + 'region': 'North Dakota', + 'locality': 'Strasburg', + 'country_a': 'USA', + 'county': 'Emmons County', + 'country': 'United States' + }, 'category': [ 'education' ], @@ -252,16 +272,18 @@ module.exports = [ 'lat': 40.597578 }, 'address': {}, - 'local_admin': 'Franklin', - 'admin1_abbr': 'OH', 'name': { 'default': 'Strasburg High School' }, - 'admin1': 'Ohio', - 'locality': 'Strasburg', - 'alpha3': 'USA', - 'admin2': 'Tuscarawas County', - 'admin0': 'United States', + 'parent': { + 'localadmin': 'Franklin', + 'region_a': 'OH', + 'region': 'Ohio', + 'locality': 'Strasburg', + 'country_a': 'USA', + 'county': 'Tuscarawas County', + 'country': 'United States' + }, 'category': [ 'education' ], diff --git a/test/unit/fixture/reverse_with_boundary_country.js b/test/unit/fixture/reverse_with_boundary_country.js index 8d739eaa..789cadaf 100644 --- a/test/unit/fixture/reverse_with_boundary_country.js +++ b/test/unit/fixture/reverse_with_boundary_country.js @@ -8,7 +8,7 @@ module.exports = { 'must': [ { 'match': { - 'alpha3': { + 'parent.country_a': { 'analyzer': 'standard', 'query': 'ABC' } diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index 3e6f83bc..5dde3344 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -7,7 +7,7 @@ module.exports = { 'must': [ { 'match': { - 'alpha3': { + 'parent.country_a': { 'analyzer': 'standard', 'query': 'ABC' } diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 316903e7..859cfda4 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -109,55 +109,55 @@ module.exports = { } }, { 'match': { - 'alpha3': { - 'query': 'USA', - 'boost': vs['admin:alpha3:boost'], - 'analyzer': vs['admin:alpha3:analyzer'] + 'parent.country': { + 'query': 'new york', + 'boost': vs['admin:country:boost'], + 'analyzer': vs['admin:country:analyzer'] } } }, { 'match': { - 'admin0': { - 'query': 'new york', - 'boost': vs['admin:admin0:boost'], - 'analyzer': vs['admin:admin0:analyzer'] + 'parent.country_a': { + 'query': 'USA', + 'boost': vs['admin:country_a:boost'], + 'analyzer': vs['admin:country_a:analyzer'] } } }, { 'match': { - 'admin1': { + 'parent.region': { 'query': 'new york', - 'boost': vs['admin:admin1:boost'], - 'analyzer': vs['admin:admin1:analyzer'] + 'boost': vs['admin:region:boost'], + 'analyzer': vs['admin:region:analyzer'] } } }, { 'match': { - 'admin1_abbr': { + 'parent.region_a': { 'query': 'NY', - 'boost': vs['admin:admin1_abbr:boost'], - 'analyzer': vs['admin:admin1_abbr:analyzer'] + 'boost': vs['admin:region_a:boost'], + 'analyzer': vs['admin:region_a:analyzer'] } } }, { 'match': { - 'admin2': { + 'parent.county': { 'query': 'new york', - 'boost': vs['admin:admin2:boost'], - 'analyzer': vs['admin:admin2:analyzer'] + 'boost': vs['admin:county:boost'], + 'analyzer': vs['admin:county:analyzer'] } } }, { 'match': { - 'local_admin': { + 'parent.localadmin': { 'query': 'new york', - 'boost': vs['admin:local_admin:boost'], - 'analyzer': vs['admin:local_admin:analyzer'] + 'boost': vs['admin:localadmin:boost'], + 'analyzer': vs['admin:localadmin:analyzer'] } } }, { 'match': { - 'locality': { + 'parent.locality': { 'query': 'new york', 'boost': vs['admin:locality:boost'], 'analyzer': vs['admin:locality:analyzer'] @@ -165,10 +165,10 @@ module.exports = { } }, { 'match': { - 'neighborhood': { + 'parent.neighbourhood': { 'query': 'new york', - 'boost': vs['admin:neighborhood:boost'], - 'analyzer': vs['admin:neighborhood:analyzer'] + 'boost': vs['admin:neighbourhood:boost'], + 'analyzer': vs['admin:neighbourhood:analyzer'] } } }] diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index 745598da..d857e31b 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -85,47 +85,47 @@ module.exports = { } },{ 'match': { - 'admin0': { + 'parent.country': { 'query': 'new york', - 'boost': vs['admin:admin0:boost'], - 'analyzer': vs['admin:admin0:analyzer'] + 'boost': vs['admin:country:boost'], + 'analyzer': vs['admin:country:analyzer'] } } }, { 'match': { - 'admin1': { + 'parent.region': { 'query': 'new york', - 'boost': vs['admin:admin1:boost'], - 'analyzer': vs['admin:admin1:analyzer'] + 'boost': vs['admin:region:boost'], + 'analyzer': vs['admin:region:analyzer'] } } }, { 'match': { - 'admin1_abbr': { + 'parent.region_a': { 'query': 'new york', - 'boost': vs['admin:admin1_abbr:boost'], - 'analyzer': vs['admin:admin1_abbr:analyzer'] + 'boost': vs['admin:region_a:boost'], + 'analyzer': vs['admin:region_a:analyzer'] } } }, { 'match': { - 'admin2': { + 'parent.county': { 'query': 'new york', - 'boost': vs['admin:admin2:boost'], - 'analyzer': vs['admin:admin2:analyzer'] + 'boost': vs['admin:county:boost'], + 'analyzer': vs['admin:county:analyzer'] } } }, { 'match': { - 'local_admin': { + 'parent.localadmin': { 'query': 'new york', - 'boost': vs['admin:local_admin:boost'], - 'analyzer': vs['admin:local_admin:analyzer'] + 'boost': vs['admin:localadmin:boost'], + 'analyzer': vs['admin:localadmin:analyzer'] } } }, { 'match': { - 'locality': { + 'parent.locality': { 'query': 'new york', 'boost': vs['admin:locality:boost'], 'analyzer': vs['admin:locality:analyzer'] @@ -133,10 +133,10 @@ module.exports = { } }, { 'match': { - 'neighborhood': { + 'parent.neighbourhood': { 'query': 'new york', - 'boost': vs['admin:neighborhood:boost'], - 'analyzer': vs['admin:neighborhood:analyzer'] + 'boost': vs['admin:neighbourhood:boost'], + 'analyzer': vs['admin:neighbourhood:analyzer'] } } }] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index 51bb57cc..25beb280 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -101,47 +101,47 @@ module.exports = { } }, { 'match': { - 'admin0': { + 'parent.country': { 'query': 'manhattan', - 'boost': vs['admin:admin0:boost'], - 'analyzer': vs['admin:admin0:analyzer'] + 'boost': vs['admin:country:boost'], + 'analyzer': vs['admin:country:analyzer'] } } }, { 'match': { - 'admin1': { + 'parent.region': { 'query': 'manhattan', - 'boost': vs['admin:admin1:boost'], - 'analyzer': vs['admin:admin1:analyzer'] + 'boost': vs['admin:region:boost'], + 'analyzer': vs['admin:region:analyzer'] } } }, { 'match': { - 'admin1_abbr': { + 'parent.region_a': { 'query': 'NY', - 'boost': vs['admin:admin1_abbr:boost'], - 'analyzer': vs['admin:admin1_abbr:analyzer'] + 'boost': vs['admin:region_a:boost'], + 'analyzer': vs['admin:region_a:analyzer'] } } }, { 'match': { - 'admin2': { + 'parent.county': { 'query': 'manhattan', - 'boost': vs['admin:admin2:boost'], - 'analyzer': vs['admin:admin2:analyzer'] + 'boost': vs['admin:county:boost'], + 'analyzer': vs['admin:county:analyzer'] } } }, { 'match': { - 'local_admin': { + 'parent.localadmin': { 'query': 'manhattan', - 'boost': vs['admin:local_admin:boost'], - 'analyzer': vs['admin:local_admin:analyzer'] + 'boost': vs['admin:localadmin:boost'], + 'analyzer': vs['admin:localadmin:analyzer'] } } }, { 'match': { - 'locality': { + 'parent.locality': { 'query': 'manhattan', 'boost': vs['admin:locality:boost'], 'analyzer': vs['admin:locality:analyzer'] @@ -149,10 +149,10 @@ module.exports = { } }, { 'match': { - 'neighborhood': { + 'parent.neighbourhood': { 'query': 'manhattan', - 'boost': vs['admin:neighborhood:boost'], - 'analyzer': vs['admin:neighborhood:analyzer'] + 'boost': vs['admin:neighbourhood:boost'], + 'analyzer': vs['admin:neighbourhood:analyzer'] } } }] diff --git a/test/unit/helper/labelGenerator_SGP.js b/test/unit/helper/labelGenerator_SGP.js index 92aca117..622f8915 100644 --- a/test/unit/helper/labelGenerator_SGP.js +++ b/test/unit/helper/labelGenerator_SGP.js @@ -14,7 +14,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.north_west_singapore = function(test, common) { test('north west singapore', function(t) { var doc = { - 'name': { 'default': 'North West' }, + 'name': 'North West', 'country_a': 'SGP', 'country': 'Singapore', 'region': 'North West' @@ -28,7 +28,7 @@ module.exports.tests.north_west_singapore = function(test, common) { module.exports.tests.singapore_mcdonalds = function(test, common) { test('singapore_mcdonalds', function(t) { var doc = { - 'name': { 'default': 'McDonald\'s' }, + 'name': 'McDonald\'s', 'country_a': 'SGP', 'country': 'Singapore', 'region': 'Central Singapore', diff --git a/test/unit/helper/labelGenerator_SWE.js b/test/unit/helper/labelGenerator_SWE.js index edefc93a..e3f8534d 100644 --- a/test/unit/helper/labelGenerator_SWE.js +++ b/test/unit/helper/labelGenerator_SWE.js @@ -14,7 +14,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.skane1 = function(test, common) { test('skåne 1', function(t) { var doc = { - 'name': { 'default': 'Malmö' }, + 'name': 'Malmö', 'country_a': 'SWE', 'country': 'Sweden', 'region': 'Skåne', @@ -29,7 +29,7 @@ module.exports.tests.skane1 = function(test, common) { module.exports.tests.skane2 = function(test, common) { test('skåne 2', function(t) { var doc = { - 'name': { 'default': 'Malmö' }, + 'name': 'Malmö', 'country_a': 'SWE', 'country': 'Sweden', 'region': 'Skåne', diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js index 7d0eb8d7..ea20c4d0 100644 --- a/test/unit/mock/backend.js +++ b/test/unit/mock/backend.js @@ -15,7 +15,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){ value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - admin0: 'country1', admin1: 'state1', admin2: 'city1' + parent: { country: 'country1', region: 'state1', county: 'city1' } } }, { _id: 'myid2', @@ -25,7 +25,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){ value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - admin0: 'country2', admin1: 'state2', admin2: 'city2' + parent: { country: 'country2', region: 'state2', county: 'city2' } } }])); }; @@ -43,7 +43,7 @@ responses['client/mget/ok/1'] = function( cmd, cb ){ value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - admin0: 'country1', admin1: 'state1', admin2: 'city1' + parent: { country: 'country1', region: 'state1', county: 'city1' } } }, { _id: 'myid2', @@ -54,7 +54,7 @@ responses['client/mget/ok/1'] = function( cmd, cb ){ value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - admin0: 'country2', admin1: 'state2', admin2: 'city2' + parent: { country: 'country2', region: 'state2', county: 'city2' } } }])); }; diff --git a/test/unit/query/search.js b/test/unit/query/search.js index b468f493..aedde407 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -126,8 +126,7 @@ module.exports.tests.query = function(test, common) { test('valid query with a full valid address', function(t) { var address = '123 main st new york ny 10010 US'; var query = generate({ text: address, - layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', - 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], + layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, parsed_text: parser.get_parsed_address(address), }); @@ -135,15 +134,20 @@ module.exports.tests.query = function(test, common) { var compiled = JSON.parse( JSON.stringify( query ) ); var expected = require('../fixture/search_full_address'); - t.deepEqual(compiled, expected, 'valid search query'); + t.deepEqual(compiled.query.filtered.query.bool.should, expected.query.filtered.query.bool.should, 'valid boundary.country query'); + + [ { match: { 'phrase.default': { analyzer: 'peliasPhrase', boost: 1, query: '123 main st', slop: 2, type: 'phrase' } } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'popularity' } }, functions: [ { field_value_factor: [Object], weight: 1 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'population' } }, functions: [ { field_value_factor: [Object], weight: 2 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { match: { 'address.number': { analyzer: 'peliasHousenumber', boost: 2, query: '123' } } }, { match: { 'address.street': { analyzer: 'peliasStreet', boost: 5, query: 'main st' } } }, { match: { 'address.zip': { analyzer: 'peliasZip', boost: 20, query: '10010' } } }, { match: { 'parent.country_a': { analyzer: 'standard', boost: 5, query: 'USA' } } }, { match: { 'parent.country': { analyzer: 'peliasAdmin', boost: 4, query: 'new york' } } }, { match: { 'parent.region': { analyzer: 'peliasAdmin', boost: 3, query: 'new york' } } }, { match: { 'parent.region_a': { analyzer: 'peliasAdmin', boost: 3, query: 'NY' } } }, { match: { 'parent.county': { analyzer: 'peliasAdmin', boost: 2, query: 'new york' } } }, { match: { 'parent.localadmin': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.locality': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.neighbourhood': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } } ] + [ { match: { 'phrase.default': { analyzer: 'peliasPhrase', boost: 1, query: '123 main st', slop: 2, type: 'phrase' } } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'popularity' } }, functions: [ { field_value_factor: [Object], weight: 1 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'population' } }, functions: [ { field_value_factor: [Object], weight: 2 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { match: { 'address.number': { analyzer: 'peliasHousenumber', boost: 2, query: '123' } } }, { match: { 'address.street': { analyzer: 'peliasStreet', boost: 5, query: 'main st' } } }, { match: { 'address.zip': { analyzer: 'peliasZip', boost: 20, query: '10010' } } }, { match: { 'parent.country': { analyzer: 'peliasAdmin', boost: 4, query: 'new york' } } }, { match: { 'parent.country_a': { analyzer: 'standard', boost: 5, query: 'USA' } } }, { match: { 'parent.region': { analyzer: 'peliasAdmin', boost: 3, query: 'new york' } } }, { match: { 'parent.region_a': { analyzer: 'peliasAdmin', boost: 3, query: 'NY' } } }, { match: { 'parent.county': { analyzer: 'peliasAdmin', boost: 2, query: 'new york' } } }, { match: { 'parent.localadmin': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.locality': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.neighbourhood': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } } ] + + + //t.deepEqual(compiled, expected, 'valid search query'); t.end(); }); test('valid query with partial address', function(t) { var partial_address = 'soho grand, new york'; var query = generate({ text: partial_address, - layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', - 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], + layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); @@ -158,8 +162,7 @@ module.exports.tests.query = function(test, common) { test('valid query with regions in address', function(t) { var partial_address = '1 water st manhattan ny'; var query = generate({ text: partial_address, - layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', - 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], + layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); diff --git a/test/unit/sanitiser/_layers.js b/test/unit/sanitiser/_layers.js index bd499fc7..a1cdd7c5 100644 --- a/test/unit/sanitiser/_layers.js +++ b/test/unit/sanitiser/_layers.js @@ -43,7 +43,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var admin_layers = [ 'continent', 'macrocountry', 'country', 'dependency', 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', - 'microhood', 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin' ]; + 'microhood', 'disputed' ]; t.deepEqual(clean.layers, admin_layers, 'coarse layers set'); t.end(); @@ -65,7 +65,7 @@ module.exports.tests.sanitize_layers = function(test, common) { sanitize(raw, clean); - var expected_layers = ['venue', 'country', 'admin0', 'region', 'admin1']; + var expected_layers = ['venue', 'country', 'region']; t.deepEqual(clean.layers, expected_layers, 'venue + regular layers'); t.end(); }); @@ -78,7 +78,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var expected_layers = [ 'continent', 'macrocountry', 'country', 'dependency', 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', - 'microhood', 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin' ]; + 'microhood', 'disputed' ]; t.deepEqual(clean.layers, expected_layers, 'coarse + regular layers set'); t.end(); @@ -90,7 +90,7 @@ module.exports.tests.sanitize_layers = function(test, common) { sanitize(raw, clean); - var expected_layers = ['address', 'country', 'admin0', 'locality' ]; + var expected_layers = ['address', 'country', 'locality' ]; t.deepEqual(clean.layers, expected_layers, 'address + regular layers set'); t.end(); }); @@ -101,7 +101,7 @@ module.exports.tests.sanitize_layers = function(test, common) { sanitize(raw, clean); - var expected_layers = ['venue', 'country', 'admin0']; + var expected_layers = ['venue', 'country']; t.deepEqual(clean.layers, expected_layers, 'venue layers found (no duplicates)'); t.end(); }); @@ -115,7 +115,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var coarse_layers = [ 'continent', 'macrocountry', 'country', 'dependency', 'region', 'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', - 'disputed', 'admin0', 'admin1', 'admin2', 'neighborhood', 'local_admin']; + 'disputed' ]; var venue_layers = [ 'venue' ]; var expected_layers = venue_layers.concat(coarse_layers); diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index d42d87f4..5d599b0f 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -72,7 +72,7 @@ module.exports.tests.sanitise_valid_text = function(test, common) { }; module.exports.tests.sanitize_text_with_delim = function(test, common) { - var texts = [ 'a,bcd', '123 main st, admin1', ',,,', ' ' ]; + var texts = [ 'a,bcd', '123 main st, region', ',,,', ' ' ]; test('valid texts with a comma', function(t) { texts.forEach( function( text ){ diff --git a/test/unit/service/mget.js b/test/unit/service/mget.js index 9f13d78b..dd093c9f 100644 --- a/test/unit/service/mget.js +++ b/test/unit/service/mget.js @@ -20,14 +20,14 @@ module.exports.tests.functional_success = function(test, common) { value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - admin0: 'country1', admin1: 'state1', admin2: 'city1' + parent: { country: 'country1', region: 'state1', county: 'city1' } }, { _id: 'myid2', _type: 'mytype2', value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - admin0: 'country2', admin1: 'state2', admin2: 'city2' + parent: { country: 'country2', region: 'state2', county: 'city2' } } ]; diff --git a/test/unit/service/search.js b/test/unit/service/search.js index e5bf2cff..37eb93af 100644 --- a/test/unit/service/search.js +++ b/test/unit/service/search.js @@ -23,7 +23,7 @@ module.exports.tests.functional_success = function(test, common) { value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - admin0: 'country1', admin1: 'state1', admin2: 'city1' + parent: { country: 'country1', region: 'state1', county: 'city1' } }, { _id: 'myid2', _type: 'mytype2', @@ -31,7 +31,7 @@ module.exports.tests.functional_success = function(test, common) { value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - admin0: 'country2', admin1: 'state2', admin2: 'city2' + parent: { country: 'country2', region: 'state2', county: 'city2' } } ]; From e9ceb25ca0fa8ef37fa699acc534105bf9ec413b Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 14:02:07 -0500 Subject: [PATCH 056/106] Fix crash when dedupe was comparing arrays as stings for parent properties --- middleware/dedupe.js | 12 +++++++++++- test/unit/query/search.js | 8 +------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index d0cce700..9e0e5c6a 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -82,7 +82,17 @@ function isDifferent(item1, item2) { * @throws {Error} */ function propMatch(item1, item2, prop) { - if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) { + var prop1 = item1[prop]; + var prop2 = item2[prop]; + + // in the case the property is an array (currently only in parent schema) + // simply take the 1st item. this will change in the near future to support multiple hierarchies + if (_.isArray(prop1) && _.isArray(prop2)) { + prop1 = prop1[0]; + prop2= prop2[0]; + } + + if (normalizeString(prop1) !== normalizeString(prop2)) { throw new Error('different'); } } diff --git a/test/unit/query/search.js b/test/unit/query/search.js index aedde407..f20122bb 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -134,13 +134,7 @@ module.exports.tests.query = function(test, common) { var compiled = JSON.parse( JSON.stringify( query ) ); var expected = require('../fixture/search_full_address'); - t.deepEqual(compiled.query.filtered.query.bool.should, expected.query.filtered.query.bool.should, 'valid boundary.country query'); - - [ { match: { 'phrase.default': { analyzer: 'peliasPhrase', boost: 1, query: '123 main st', slop: 2, type: 'phrase' } } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'popularity' } }, functions: [ { field_value_factor: [Object], weight: 1 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'population' } }, functions: [ { field_value_factor: [Object], weight: 2 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { match: { 'address.number': { analyzer: 'peliasHousenumber', boost: 2, query: '123' } } }, { match: { 'address.street': { analyzer: 'peliasStreet', boost: 5, query: 'main st' } } }, { match: { 'address.zip': { analyzer: 'peliasZip', boost: 20, query: '10010' } } }, { match: { 'parent.country_a': { analyzer: 'standard', boost: 5, query: 'USA' } } }, { match: { 'parent.country': { analyzer: 'peliasAdmin', boost: 4, query: 'new york' } } }, { match: { 'parent.region': { analyzer: 'peliasAdmin', boost: 3, query: 'new york' } } }, { match: { 'parent.region_a': { analyzer: 'peliasAdmin', boost: 3, query: 'NY' } } }, { match: { 'parent.county': { analyzer: 'peliasAdmin', boost: 2, query: 'new york' } } }, { match: { 'parent.localadmin': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.locality': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.neighbourhood': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } } ] - [ { match: { 'phrase.default': { analyzer: 'peliasPhrase', boost: 1, query: '123 main st', slop: 2, type: 'phrase' } } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'popularity' } }, functions: [ { field_value_factor: [Object], weight: 1 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { function_score: { boost_mode: 'replace', filter: { exists: { field: 'population' } }, functions: [ { field_value_factor: [Object], weight: 2 } ], max_boost: 20, query: { match: { 'phrase.default': [Object] } }, score_mode: 'first' } }, { match: { 'address.number': { analyzer: 'peliasHousenumber', boost: 2, query: '123' } } }, { match: { 'address.street': { analyzer: 'peliasStreet', boost: 5, query: 'main st' } } }, { match: { 'address.zip': { analyzer: 'peliasZip', boost: 20, query: '10010' } } }, { match: { 'parent.country': { analyzer: 'peliasAdmin', boost: 4, query: 'new york' } } }, { match: { 'parent.country_a': { analyzer: 'standard', boost: 5, query: 'USA' } } }, { match: { 'parent.region': { analyzer: 'peliasAdmin', boost: 3, query: 'new york' } } }, { match: { 'parent.region_a': { analyzer: 'peliasAdmin', boost: 3, query: 'NY' } } }, { match: { 'parent.county': { analyzer: 'peliasAdmin', boost: 2, query: 'new york' } } }, { match: { 'parent.localadmin': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.locality': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } }, { match: { 'parent.neighbourhood': { analyzer: 'peliasAdmin', boost: 1, query: 'new york' } } } ] - - - //t.deepEqual(compiled, expected, 'valid search query'); + t.deepEqual(compiled, expected, 'valid search query'); t.end(); }); From b7bc3593c2edbfaa677916de36dd1b73ce194bb2 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 14:06:08 -0500 Subject: [PATCH 057/106] Lint cleanup --- middleware/renamePlacenames.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index f3e35829..4d1f5248 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -68,7 +68,7 @@ function renameStuff(place) { if (place.parent) { PARENT_PROPS.forEach(function (prop) { place[prop] = place.parent[prop]; - }) + }); } return place; From b3939efb216ce5885a91860ae37d1185a420a84b Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 14:07:03 -0500 Subject: [PATCH 058/106] Fix ciao tests --- test/ciao/reverse/layers_alias_coarse.coffee | 7 +------ test/ciao/reverse/layers_invalid.coffee | 2 +- test/ciao/reverse/layers_mix_invalid_valid.coffee | 2 +- test/ciao/reverse/layers_multiple.coffee | 2 +- test/ciao/reverse/layers_single.coffee | 2 +- test/ciao/reverse/sources_invalid.coffee | 2 +- test/ciao/reverse/sources_layers_invalid_combo.coffee | 6 +++--- test/ciao/search/layers_alias_coarse.coffee | 7 +------ test/ciao/search/layers_invalid.coffee | 2 +- test/ciao/search/layers_mix_invalid_valid.coffee | 2 +- test/ciao/search/layers_multiple.coffee | 2 +- test/ciao/search/layers_single.coffee | 2 +- test/ciao/search/sources_invalid.coffee | 2 +- test/ciao/search/sources_layers_invalid_combo.coffee | 6 +++--- test/unit/sanitiser/_sources.js | 4 ++-- 15 files changed, 20 insertions(+), 30 deletions(-) diff --git a/test/ciao/reverse/layers_alias_coarse.coffee b/test/ciao/reverse/layers_alias_coarse.coffee index 6b911083..f16f5a17 100644 --- a/test/ciao/reverse/layers_alias_coarse.coffee +++ b/test/ciao/reverse/layers_alias_coarse.coffee @@ -41,10 +41,5 @@ json.geocoding.query.layers.should.eql [ "continent", "macrohood", "neighbourhood", "microhood", - "disputed", - "admin0", - "admin1", - "admin2", - "neighborhood", - "local_admin" + "disputed" ] diff --git a/test/ciao/reverse/layers_invalid.coffee b/test/ciao/reverse/layers_invalid.coffee index a7105815..9d89e05a 100644 --- a/test/ciao/reverse/layers_invalid.coffee +++ b/test/ciao/reverse/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_mix_invalid_valid.coffee b/test/ciao/reverse/layers_mix_invalid_valid.coffee index f8e07adf..3e6a57fe 100644 --- a/test/ciao/reverse/layers_mix_invalid_valid.coffee +++ b/test/ciao/reverse/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_multiple.coffee b/test/ciao/reverse/layers_multiple.coffee index d85c07a2..75b466d2 100644 --- a/test/ciao/reverse/layers_multiple.coffee +++ b/test/ciao/reverse/layers_multiple.coffee @@ -30,4 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.layers.should.eql ["country","admin0","region","admin1"] +json.geocoding.query.layers.should.eql ["country","region"] diff --git a/test/ciao/reverse/layers_single.coffee b/test/ciao/reverse/layers_single.coffee index ab1f8e08..8f5220eb 100644 --- a/test/ciao/reverse/layers_single.coffee +++ b/test/ciao/reverse/layers_single.coffee @@ -30,4 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.layers.should.eql ["country","admin0"] +json.geocoding.query.layers.should.eql ["country"] diff --git a/test/ciao/reverse/sources_invalid.coffee b/test/ciao/reverse/sources_invalid.coffee index f0b54803..62a6c4c2 100644 --- a/test/ciao/reverse/sources_invalid.coffee +++ b/test/ciao/reverse/sources_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' ] +json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/sources_layers_invalid_combo.coffee b/test/ciao/reverse/sources_layers_invalid_combo.coffee index 984830a7..baf157cc 100644 --- a/test/ciao/reverse/sources_layers_invalid_combo.coffee +++ b/test/ciao/reverse/sources_layers_invalid_combo.coffee @@ -1,6 +1,6 @@ #> sources and layers specified (invalid combo) -path: '/v1/reverse?point.lat=1&point.lon=2&sources=quattroshapes&layers=address' +path: '/v1/reverse?point.lat=1&point.lon=2&sources=whosonfirst&layers=address' #? 200 ok response.statusCode.should.be.equal 400 @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the quattroshapes source has nothing in the address layer' ] +json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ] #? expected warnings should.not.exist json.geocoding.warnings @@ -32,4 +32,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 json.geocoding.query.layers.should.eql ["address"] -json.geocoding.query.sources.should.eql ["quattroshapes"] +json.geocoding.query.sources.should.eql ["whosonfirst"] diff --git a/test/ciao/search/layers_alias_coarse.coffee b/test/ciao/search/layers_alias_coarse.coffee index c2527df0..9dda2a48 100644 --- a/test/ciao/search/layers_alias_coarse.coffee +++ b/test/ciao/search/layers_alias_coarse.coffee @@ -42,10 +42,5 @@ json.geocoding.query.layers.should.eql [ "continent", "macrohood", "neighbourhood", "microhood", - "disputed", - "admin0", - "admin1", - "admin2", - "neighborhood", - "local_admin" + "disputed" ] diff --git a/test/ciao/search/layers_invalid.coffee b/test/ciao/search/layers_invalid.coffee index 40de3f85..5d07f99c 100644 --- a/test/ciao/search/layers_invalid.coffee +++ b/test/ciao/search/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_mix_invalid_valid.coffee b/test/ciao/search/layers_mix_invalid_valid.coffee index 6f1ccec6..0a13c70b 100644 --- a/test/ciao/search/layers_mix_invalid_valid.coffee +++ b/test/ciao/search/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,country,region,address,venue,county,locality,admin0,admin1,admin2,neighborhood,local_admin,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_multiple.coffee b/test/ciao/search/layers_multiple.coffee index c735a206..2555573e 100644 --- a/test/ciao/search/layers_multiple.coffee +++ b/test/ciao/search/layers_multiple.coffee @@ -31,4 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.layers.should.eql ["country","admin0","region","admin1"] +json.geocoding.query.layers.should.eql ["country","region"] diff --git a/test/ciao/search/layers_single.coffee b/test/ciao/search/layers_single.coffee index ad104fab..e1931c04 100644 --- a/test/ciao/search/layers_single.coffee +++ b/test/ciao/search/layers_single.coffee @@ -31,4 +31,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 -json.geocoding.query.layers.should.eql ["country","admin0"] +json.geocoding.query.layers.should.eql ["country"] diff --git a/test/ciao/search/sources_invalid.coffee b/test/ciao/search/sources_invalid.coffee index 8b6b9d27..1620d7ba 100644 --- a/test/ciao/search/sources_invalid.coffee +++ b/test/ciao/search/sources_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' ] +json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/sources_layers_invalid_combo.coffee b/test/ciao/search/sources_layers_invalid_combo.coffee index d1e0fc07..821bf243 100644 --- a/test/ciao/search/sources_layers_invalid_combo.coffee +++ b/test/ciao/search/sources_layers_invalid_combo.coffee @@ -1,6 +1,6 @@ #> sources and layers specified (invalid combo) -path: '/v1/search?text=a&sources=quattroshapes&layers=address' +path: '/v1/search?text=a&sources=whosonfirst&layers=address' #? 200 ok response.statusCode.should.be.equal 400 @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the quattroshapes source has nothing in the address layer' ] +json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ] #? expected warnings should.not.exist json.geocoding.warnings @@ -33,5 +33,5 @@ should.not.exist json.geocoding.warnings json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 json.geocoding.query.layers.should.eql ["address"] -json.geocoding.query.sources.should.eql ["quattroshapes"] +json.geocoding.query.sources.should.eql ["whosonfirst"] should.not.exist json.geocoding.query['type'] diff --git a/test/unit/sanitiser/_sources.js b/test/unit/sanitiser/_sources.js index 61b2be90..b0daffef 100644 --- a/test/unit/sanitiser/_sources.js +++ b/test/unit/sanitiser/_sources.js @@ -29,7 +29,7 @@ module.exports.tests.no_sources = function(test, common) { }; var expected_error = 'sources parameter cannot be an empty string. ' + - 'Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst'; + 'Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst'; var messages = sanitize(req.query, req.clean); @@ -104,7 +104,7 @@ module.exports.tests.invalid_sources = function(test, common) { var expected_messages = { errors: [ '\'notasource\' is an invalid sources parameter. ' + - 'Valid options: osm,oa,gn,qs,wof,openstreetmap,openaddresses,geonames,quattroshapes,whosonfirst' + 'Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ], warnings: [] }; From b2accb71e9ee37407b231a4a0ee9df9dea99b09b Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 16:54:02 -0500 Subject: [PATCH 059/106] Get rid of QS references --- helper/type_mapping.js | 3 +-- public/attribution.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/helper/type_mapping.js b/helper/type_mapping.js index f2e49d8f..39abb835 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -17,7 +17,7 @@ function addStandardTargetsToAliases(standard, aliases) { */ // a list of all sources -var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'quattroshapes', 'whosonfirst']; +var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'whosonfirst']; /* * A list of alternate names for sources, mostly used to save typing @@ -26,7 +26,6 @@ var SOURCE_ALIASES = { 'osm': ['openstreetmap'], 'oa': ['openaddresses'], 'gn': ['geonames'], - 'qs': ['quattroshapes'], 'wof': ['whosonfirst'] }; diff --git a/public/attribution.md b/public/attribution.md index 2302c093..732e20d4 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -3,5 +3,5 @@ * Data from * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation - * [Quattroshapes](https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md) under [CC-BY-2.0](https://creativecommons.org/licenses/by/2.0/) * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/) + * [WhosOnFirst](http://whosonfirst.mapzen.com) From 4dc1da27a554462b426015db55a1e36871525d0b Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 29 Feb 2016 22:35:28 -0500 Subject: [PATCH 060/106] Upgrade pelias-query to 6.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 180576f6..8ba9782f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "pelias-config": "^1.0.1", "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", - "pelias-query": "^5.1.0", + "pelias-query": "^6.0.0", "pelias-schema": "3.0.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", From aebaf52f61fbb1434eaa8be1463783b82f1ea738 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 16:03:20 -0500 Subject: [PATCH 061/106] Remove pelias-schema from dependencies --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 8ba9782f..8e576b42 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", "pelias-query": "^6.0.0", - "pelias-schema": "3.0.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", "through2": "2.0.1" From a2950eebc4816e552b97e0e03e3603efc7f0f916 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 16:21:35 -0500 Subject: [PATCH 062/106] Cleanup renamePlacenames --- middleware/renamePlacenames.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 4d1f5248..8f171e24 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -40,24 +40,24 @@ var PARENT_PROPS = [ function setup() { - return renamePlacenames; } function renamePlacenames(req, res, next) { - // do nothing if no result data set if (!res || !res.data) { return next(); } - res.data = res.data.map(renameStuff); - + res.data = res.data.map(renameOneRecord); + next(); } -function renameStuff(place) { - +/* + * Rename the fields in one record + */ +function renameOneRecord(place) { if (place.address) { Object.keys(ADDRESS_PROPS).forEach(function (prop) { place[ADDRESS_PROPS[prop]] = place.address[prop]; From 40d89d796066bfb1b10c9d8674fbf97773c5208a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 2 Mar 2016 16:24:35 -0500 Subject: [PATCH 063/106] Add link to WOF license file --- public/attribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/attribution.md b/public/attribution.md index 732e20d4..0eedb365 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -4,4 +4,4 @@ * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/) - * [WhosOnFirst](http://whosonfirst.mapzen.com) + * [WhosOnFirst](http://whosonfirst.mapzen.com) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md) From 3a8539286297396890309713b4df581c40ac0683 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 3 Mar 2016 10:18:13 -0500 Subject: [PATCH 064/106] Update _type value in ES response fixtures osmnode and osmway are not types anymore! --- test/unit/fixture/dedupe_elasticsearch_results.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index 6f3d6d6c..84b279d3 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -22,7 +22,7 @@ module.exports = [ 'education' ], '_id': '357321757', - '_type': 'osmnode', + '_type': 'venue', '_score': 1.2367082, 'confidence': 0.879 }, @@ -49,7 +49,7 @@ module.exports = [ 'education' ], '_id': '357321757', - '_type': 'osmnode', + '_type': 'venue', '_score': 1.2367082, 'confidence': 0.879 }, @@ -130,7 +130,7 @@ module.exports = [ 'education' ], '_id': '357294404', - '_type': 'osmnode', + '_type': 'venue', '_score': 1.2367082, 'confidence': 0.879 }, @@ -157,7 +157,7 @@ module.exports = [ 'education' ], '_id': '357283977', - '_type': 'osmnode', + '_type': 'venue', '_score': 1.1036991, 'confidence': 0.664 }, @@ -210,7 +210,7 @@ module.exports = [ 'education' ], '_id': '358058986', - '_type': 'osmnode', + '_type': 'venue', '_score': 1.0492544, 'confidence': 0.658 }, @@ -288,8 +288,8 @@ module.exports = [ 'education' ], '_id': '356646971', - '_type': 'osmway', + '_type': 'venue', '_score': 0.9724125, 'confidence': 0.649 } -]; \ No newline at end of file +]; From cd16ff92b1c53893eec30e9e6f501220578115ef Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 9 Mar 2016 14:10:01 +0100 Subject: [PATCH 065/106] add common.diff() function for test debugging --- package.json | 1 + test/unit/run.js | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 180576f6..17a5c074 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ }, "devDependencies": { "ciao": "^0.6.0", + "difflet": "^1.0.1", "istanbul": "^0.4.2", "jshint": "^2.5.6", "nsp": "^2.2.0", diff --git a/test/unit/run.js b/test/unit/run.js index 9d210853..de13bbe0 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -1,6 +1,12 @@ +var tape = require('tape'), + diff = require('difflet')({ indent : 2, comment : true }); -var tape = require('tape'); -var common = {}; +var common = { + // a visual deep diff rendered using console.error() + diff: function( actual, expected ){ + console.error( diff.compare( actual, expected ) ); + } +}; var tests = [ require('./controller/index'), From ab427229f484d5c28b9f59e3074da515acfc7fc0 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 5 Feb 2016 19:55:28 -0500 Subject: [PATCH 066/106] Use the official elasticsearch This replaces the pelias-esclient library which has not been maintained for a long time. --- package.json | 2 +- src/backend.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 5180e5a0..d44b0576 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "async": "^1.5.2", "check-types": "^6.0.0", "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", + "elasticsearch": "^10.1.3", "express": "^4.8.8", "express-http-proxy": "^0.6.0", "extend": "3.0.0", @@ -51,7 +52,6 @@ "markdown": "0.5.0", "morgan": "1.7.0", "pelias-config": "^1.0.1", - "pelias-esclient": "1.0.0", "pelias-logger": "^0.0.8", "pelias-query": "^6.0.0", "pelias-suggester-pipeline": "2.0.4", diff --git a/src/backend.js b/src/backend.js index d0388966..3d2bc993 100644 --- a/src/backend.js +++ b/src/backend.js @@ -1,6 +1,6 @@ var Backend = require('geopipes-elasticsearch-backend'), - client = require('pelias-esclient')(), + client = require('elasticsearch').Client(), backends = {}; function getBackend( index, type ){ @@ -11,4 +11,4 @@ function getBackend( index, type ){ return backends[key]; } -module.exports = getBackend; \ No newline at end of file +module.exports = getBackend; From 36b6adddfdda5a958be93cadfb5f800c69170895 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Wed, 9 Mar 2016 18:16:26 -0500 Subject: [PATCH 067/106] Add bounding_box parsing step --- middleware/parseBBox.js | 38 ++++++++++++++++++ routes/v1.js | 7 +++- test/unit/middleware/parseBBox.js | 65 +++++++++++++++++++++++++++++++ test/unit/run.js | 1 + 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 middleware/parseBBox.js create mode 100644 test/unit/middleware/parseBBox.js diff --git a/middleware/parseBBox.js b/middleware/parseBBox.js new file mode 100644 index 00000000..990b3498 --- /dev/null +++ b/middleware/parseBBox.js @@ -0,0 +1,38 @@ +var logger = require('pelias-logger').get('api'); + +/** + * Parses the bounding box property in docs, if one is found + */ + +function setup() { + return function (req, res, next) { + // do nothing if no result data set + if (!res || !res.data) { + return next(); + } + + res.data = res.data.map(parseBBox); + + next(); + }; +} + +/* + * Parse the bbox property and form an object + */ +function parseBBox(place) { + + if (place && place.bounding_box) { + try { + place.bounding_box = JSON.parse(place.bounding_box); + } + catch (err) { + logger.error('Invalid bounding_box json string:', place); + delete place.bounding_box; + } + } + + return place; +} + +module.exports = setup; diff --git a/routes/v1.js b/routes/v1.js index 00c99aa5..eacd0a40 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -34,7 +34,8 @@ var postProc = { localNamingConventions: require('../middleware/localNamingConventions'), renamePlacenames: require('../middleware/renamePlacenames'), geocodeJSON: require('../middleware/geocodeJSON'), - sendJSON: require('../middleware/sendJSON') + sendJSON: require('../middleware/sendJSON'), + parseBoundingBox: require('../middleware/parseBBox') }; /** @@ -65,6 +66,7 @@ function addRoutes(app, peliasConfig) { postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), + postProc.parseBoundingBox(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -76,6 +78,7 @@ function addRoutes(app, peliasConfig) { postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), + postProc.parseBoundingBox(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -90,6 +93,7 @@ function addRoutes(app, peliasConfig) { postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), + postProc.parseBoundingBox(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -98,6 +102,7 @@ function addRoutes(app, peliasConfig) { controllers.place(), postProc.localNamingConventions(), postProc.renamePlacenames(), + postProc.parseBoundingBox(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), diff --git a/test/unit/middleware/parseBBox.js b/test/unit/middleware/parseBBox.js new file mode 100644 index 00000000..4e2cd802 --- /dev/null +++ b/test/unit/middleware/parseBBox.js @@ -0,0 +1,65 @@ +var parseBBox = require('../../../middleware/parseBBox')(); + +module.exports.tests = {}; + +module.exports.tests.computeDistance = function(test, common) { + test('valid bounding_box json', function(t) { + var res = { + data: [ + { + bounding_box: '{"min_lat":40.6514712164,"max_lat":40.6737320588,"min_lon":-73.8967895508,"max_lon":-73.8665771484}' + } + ] + }; + + var expected = { + data: [ + { + bounding_box: { + min_lat: 40.6514712164, + max_lat: 40.6737320588, + min_lon: -73.8967895508, + max_lon: -73.8665771484 + } + } + ] + }; + + parseBBox({}, res, function () { + t.deepEquals(res, expected, 'correct bounding_box'); + t.end(); + }); + }); + + test('invalid bounding_box json', function(t) { + var res = { + data: [ + { + bounding_box: 'garbage json' + } + ] + }; + + var expected = { + data: [ + {} + ] + }; + + parseBBox({}, res, function () { + t.deepEquals(res, expected, 'correct bounding_box'); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('[middleware] parseBBox: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/run.js b/test/unit/run.js index de13bbe0..0ce25e7d 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -27,6 +27,7 @@ var tests = [ require('./middleware/distance'), require('./middleware/localNamingConventions'), require('./middleware/dedupe'), + require('./middleware/parseBBox'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'), From e17b940136cd25e92c04c33f9302d14a97746256 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Wed, 9 Mar 2016 18:18:00 -0500 Subject: [PATCH 068/106] Compute extent bounding box and expose bounding_box property in results --- helper/geojsonify.js | 51 ++++++++++++-- test/unit/helper/geojsonify.js | 125 +++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 28fa6272..8910767d 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -29,7 +29,8 @@ var DETAILS_PROPS = [ 'locality_id', 'locality_a', 'neighbourhood', - 'neighbourhood_id' + 'neighbourhood_id', + 'bounding_box' ]; @@ -50,11 +51,16 @@ function geojsonifyPlaces( docs ){ return !!doc; }); + // get all the bounding_box corners as well as single points + // to be used for computing the overall bounding_box for the FeatureCollection + var extentPoints = extractExtentPoints(geodata); + // convert to geojson - var geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] }); + var geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] }); + var geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] }); // bounding box calculations - computeBBox(geojson); + computeBBox(geojson, geojsonExtentPoints); return geojson; } @@ -105,23 +111,56 @@ function addLabel(src, dst) { dst.label = labelGenerator(dst); } + +/** + * Collect all points from the geodata. + * If an item is a single point, just use that. + * If an item has a bounding box, add two corners of the box as individual points. + * + * @param {Array} geodata + * @returns {Array} + */ +function extractExtentPoints(geodata) { + var extentPoints = []; + geodata.forEach(function (place) { + if (place.bounding_box) { + extentPoints.push({ + lng: place.bounding_box.min_lon, + lat: place.bounding_box.min_lat + }); + extentPoints.push({ + lng: place.bounding_box.max_lon, + lat: place.bounding_box.max_lat + }); + } + else { + extentPoints.push({ + lng: place.lng, + lat: place.lat + }); + } + }); + + return extentPoints; +} + /** * Compute bbox that encompasses all features in the result set. * Set bbox property on the geojson object. * * @param {object} geojson */ -function computeBBox(geojson) { +function computeBBox(geojson, geojsonExtentPoints) { // @note: extent() sometimes throws Errors for unusual data // eg: https://github.com/pelias/pelias/issues/84 try { - var bbox = extent( geojson ); + var bbox = extent( geojsonExtentPoints ); if( !!bbox ){ geojson.bbox = bbox; } } catch( e ){ console.error( 'bbox error', e.message, e.stack ); - console.error( 'geojson', JSON.stringify( geojson, null, 2 ) ); + console.error( 'geojson', JSON.stringify( geojsonExtentPoints, null, 2 ) ); } } diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 3ab4d975..0aa6b39c 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -207,6 +207,131 @@ module.exports.tests.search = function(test, common) { t.deepEqual(json, expected, 'all docs mapped'); t.end(); }); + + test('filtering out empty items', function (t) { + var input = [ + { + 'bounding_box': { + 'min_lat': 40.6514712164, + 'max_lat': 40.6737320588, + 'min_lon': -73.8967895508, + 'max_lon': -73.8665771484 + }, + 'locality': [ + 'New York' + ], + 'source': 'whosonfirst', + 'layer': 'neighbourhood', + 'population': 173198, + 'popularity': 495, + 'center_point': { + 'lon': -73.881319, + 'lat': 40.663303 + }, + 'name': { + 'default': 'East New York' + }, + 'source_id': '85816607', + 'category': [], + '_id': '85816607', + '_type': 'neighbourhood', + '_score': 21.434, + 'confidence': 0.888, + 'country': [ + 'United States' + ], + 'country_id': [ + '85633793' + ], + 'country_a': [ + 'USA' + ], + 'region': [ + 'New York' + ], + 'region_id': [ + '85688543' + ], + 'region_a': [ + 'NY' + ], + 'county': [ + 'Kings County' + ], + 'county_id': [ + '102082361' + ], + 'county_a': [ + null + ], + 'localadmin': [ + 'Brooklyn' + ], + 'localadmin_id': [ + '404521211' + ], + 'localadmin_a': [ + null + ], + 'locality_id': [ + '85977539' + ], + 'locality_a': [ + null + ], + 'neighbourhood': [], + 'neighbourhood_id': [] + } + ]; + + var expected = { + 'type': 'FeatureCollection', + 'bbox': [-73.8967895508, 40.6514712164, -73.8665771484, 40.6737320588], + 'features': [ + { + 'type': 'Feature', + 'properties': { + 'id': '85816607', + 'gid': 'whosonfirst:neighbourhood:85816607', + 'layer': 'neighbourhood', + 'source': 'whosonfirst', + 'name': 'East New York', + 'confidence': 0.888, + 'country': 'United States', + 'country_id': '85633793', + 'country_a': 'USA', + 'region': 'New York', + 'region_id': '85688543', + 'region_a': 'NY', + 'county': 'Kings County', + 'county_id': '102082361', + 'localadmin': 'Brooklyn', + 'localadmin_id': '404521211', + 'locality': 'New York', + 'locality_id': '85977539', + 'bounding_box': { + 'min_lat': 40.6514712164, + 'max_lat': 40.6737320588, + 'min_lon': -73.8967895508, + 'max_lon': -73.8665771484 + }, + 'label': 'East New York, Brooklyn, NY, USA' + }, + 'geometry': { + 'type': 'Point', + 'coordinates': [ + -73.881319, + 40.663303 + ] + } + } + ] + }; + + var json = geojsonify.search( input ); + t.deepEqual(json, expected, 'all wanted properties exposed'); + t.end(); + }); }; module.exports.all = function (tape, common) { From 0e6bf8ed00abdbc82d023cfcaffc2f9d48403f7b Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 10 Mar 2016 09:07:22 +0200 Subject: [PATCH 069/106] Improve response deduping Consider locality and neighborhood, too. Do not take absence of an attribute as a difference. --- middleware/dedupe.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 9e0e5c6a..cd780bdf 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -42,13 +42,17 @@ function isDifferent(item1, item2) { if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) { propMatch(item1.parent, item2.parent, 'region_a'); propMatch(item1.parent, item2.parent, 'country'); + propMatch(item1.parent, item2.parent, 'locality'); + propMatch(item1.parent, item2.parent, 'neighborhood'); } else if (item1.parent !== item2.parent) { throw new Error('different'); } if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) { - propMatch(item1.name, item2.name, 'default'); + for (var lang in item1.name) { + propMatch(item1.name, item2.name, lang); + } } else { propMatch(item1, item2, 'name'); @@ -92,6 +96,10 @@ function propMatch(item1, item2, prop) { prop2= prop2[0]; } + if (!prop1 || !prop2) { + return; // absence of information does not make an item different + } + if (normalizeString(prop1) !== normalizeString(prop2)) { throw new Error('different'); } @@ -111,4 +119,4 @@ function normalizeString(str) { } -module.exports = setup; \ No newline at end of file +module.exports = setup; From 6430930cb5203155bc9e4c5a54ce0b4b103b1916 Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 10 Mar 2016 09:34:41 +0200 Subject: [PATCH 070/106] Dedupe unit test data for testing missing attributes Test data has two otherwise identical entries, but the latter one lacks some attributes. It should be considered as a duplicate. --- .../fixture/dedupe_elasticsearch_results.js | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index 84b279d3..b332a29f 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -291,5 +291,28 @@ module.exports = [ '_type': 'venue', '_score': 0.9724125, 'confidence': 0.649 + }, + { // same as above, but with some missing attributes. Should be considered as duplicate. + 'center_point': { + 'lon': -81.532392, + 'lat': 40.597578 + }, + 'address': {}, + 'name': { + 'default': 'Strasburg High School' + }, + 'parent': { + 'localadmin': 'Franklin', + 'region_a': 'OH', + 'region': 'Ohio' + }, + 'category': [ + 'education' + ], + '_id': '356646971', + '_type': 'venue', + '_score': 0.9724125, + 'confidence': 0.649 } + ]; From 827788e16c90f72f88ab375c020fd0b81397c3ee Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 10 Mar 2016 10:01:52 +0200 Subject: [PATCH 071/106] Revert "Dedupe unit test data for testing missing attributes" This reverts commit 6430930cb5203155bc9e4c5a54ce0b4b103b1916. --- .../fixture/dedupe_elasticsearch_results.js | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index b332a29f..84b279d3 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -291,28 +291,5 @@ module.exports = [ '_type': 'venue', '_score': 0.9724125, 'confidence': 0.649 - }, - { // same as above, but with some missing attributes. Should be considered as duplicate. - 'center_point': { - 'lon': -81.532392, - 'lat': 40.597578 - }, - 'address': {}, - 'name': { - 'default': 'Strasburg High School' - }, - 'parent': { - 'localadmin': 'Franklin', - 'region_a': 'OH', - 'region': 'Ohio' - }, - 'category': [ - 'education' - ], - '_id': '356646971', - '_type': 'venue', - '_score': 0.9724125, - 'confidence': 0.649 } - ]; From caaee361b2b9e3523897f3754c95ccd1befdf3d4 Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 10 Mar 2016 10:04:10 +0200 Subject: [PATCH 072/106] Take absence of information as a difference, after all Is '5 main street' same as 'main street'? Probably not. It may be that less detailed data is different data, not just bad data. maybe this can be changed if coordinates are considered, too. --- middleware/dedupe.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index cd780bdf..5f65e54b 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -96,10 +96,6 @@ function propMatch(item1, item2, prop) { prop2= prop2[0]; } - if (!prop1 || !prop2) { - return; // absence of information does not make an item different - } - if (normalizeString(prop1) !== normalizeString(prop2)) { throw new Error('different'); } From aff146aec3a0f76b0309f5ac41e07b1370949a84 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 10 Mar 2016 15:59:35 -0500 Subject: [PATCH 073/106] Fix local naming convention and confidence score middlewares --- middleware/confidenceScore.js | 10 +- middleware/localNamingConventions.js | 2 +- test/unit/controller/search.js | 12 +- .../dedupe_elasticsearch_nonascii_results.js | 30 ++-- .../fixture/dedupe_elasticsearch_results.js | 168 +++++++++--------- test/unit/middleware/confidenceScore.js | 12 +- .../unit/middleware/localNamingConventions.js | 18 +- test/unit/mock/backend.js | 8 +- test/unit/service/mget.js | 4 +- test/unit/service/search.js | 4 +- 10 files changed, 134 insertions(+), 134 deletions(-) diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index 3cd9d1f3..a0382b25 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -95,7 +95,7 @@ function checkForDealBreakers(req, hit) { return false; } - if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.parent.region_a) { + if (check.assigned(req.clean.parsed_text.state) && req.clean.parsed_text.state !== hit.parent.region_a[0]) { logger.debug('[confidence][deal-breaker]: state !== region_a'); return true; } @@ -210,8 +210,8 @@ function propMatch(textProp, hitProp, expectEnriched) { * @param {string|number} [hit.address.number] * @param {string} [hit.address.street] * @param {string|number} [hit.address.zip] - * @param {string} [hit.parent.region_a] - * @param {string} [hit.parent.country_a] + * @param {Array} [hit.parent.region_a] + * @param {Array} [hit.parent.country_a] * @returns {number} */ function checkAddress(text, hit) { @@ -222,8 +222,8 @@ function checkAddress(text, hit) { res += propMatch(text.number, (hit.address ? hit.address.number : null), false); res += propMatch(text.street, (hit.address ? hit.address.street : null), false); res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true); - res += propMatch(text.state, hit.parent.region_a, true); - res += propMatch(text.country, hit.parent.country_a, true); + res += propMatch(text.state, hit.parent.region_a[0], true); + res += propMatch(text.country, hit.parent.country_a[0], true); res /= checkCount; } diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 084cfc26..6170cda2 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -13,7 +13,7 @@ function applyLocalNamingConventions(req, res, next) { // loop through data items and flip relevant number/street res.data.filter(function(place){ // only relevant for German addresses - if( 'DEU' !== place.parent.country_a ){ return false; } + if( place.parent.country_a.indexOf('DEU') === -1 ){ return false; } if( !place.hasOwnProperty('address') ){ return false; } if( !place.address.hasOwnProperty('number') ){ return false; } if( !place.address.hasOwnProperty('street') ){ return false; } diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js index 3d46dde0..cfb715fa 100644 --- a/test/unit/controller/search.js +++ b/test/unit/controller/search.js @@ -51,9 +51,9 @@ module.exports.tests.functional_success = function(test, common) { _score: 10, _type: 'mytype1', parent: { - country: 'country1', - region: 'state1', - county: 'city1' + country: ['country1'], + region: ['state1'], + county: ['city1'] }, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, @@ -64,9 +64,9 @@ module.exports.tests.functional_success = function(test, common) { _score: 20, _type: 'mytype2', parent: { - country: 'country2', - region: 'state2', - county: 'city2' + country: ['country2'], + region: ['state2'], + county: ['city2'] }, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, diff --git a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js index ce929152..b2a6c383 100644 --- a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js @@ -8,9 +8,9 @@ module.exports = [ 'default': '万里长城万里长城' }, 'parent': { - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Beijing' + 'country_a': ['CHN'], + 'country': ['China'], + 'region': ['Beijing'] }, 'confidence': 0.733 }, @@ -23,9 +23,9 @@ module.exports = [ 'default': '万里长城' }, 'parent': { - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Beijing' + 'country_a': ['CHN'], + 'country': ['China'], + 'region': ['Beijing'] }, 'confidence': 0.733 }, @@ -38,12 +38,12 @@ module.exports = [ 'default': '万里花' }, 'parent': { - 'country_a': 'JPN', - 'country': 'Japan', - 'region': 'Tokyo', - 'county': '豊島区', - 'locality': 'Tokyo', - 'neighbourhood': '２丁目' + 'country_a': ['JPN'], + 'country': ['Japan'], + 'region': ['Tokyo'], + 'county': ['豊島区'], + 'locality': ['Tokyo'], + 'neighbourhood': ['２丁目'] }, 'confidence': 0.646 }, @@ -60,9 +60,9 @@ module.exports = [ 'postalcode': '312044' }, 'parent': { - 'country_a': 'CHN', - 'country': 'China', - 'region': 'Zhejiang' + 'country_a': ['CHN'], + 'country': ['China'], + 'region': ['Zhejiang'] }, 'confidence': 0.646 } diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index 84b279d3..a57a3a7f 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -6,14 +6,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'localadmin': 'East Lampeter', - 'region_a': 'PA', - 'region': 'Pennsylvania', - 'locality': 'Smoketown', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Greenland' + 'localadmin': ['East Lampeter'], + 'region_a': ['PA'], + 'region': ['Pennsylvania'], + 'locality': ['Smoketown'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Greenland'] }, 'name': { 'default': 'East Lampeter High School' @@ -33,14 +33,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'localadmin': 'East Lampeter', - 'region_a': 'PA', - 'region': 'Pennsylvania', - 'locality': 'Smoketown', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Greenland' + 'localadmin': ['East Lampeter'], + 'region_a': ['PA'], + 'region': ['Pennsylvania'], + 'locality': ['Smoketown'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Greenland'] }, 'name': { 'default': 'East Lampeter, High-School' @@ -60,14 +60,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'localadmin': 'West Lampeter', - 'region_a': 'PA', - 'region': 'Pennsylvania', - 'locality': 'Lampeter', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Wheatland Mills' + 'localadmin': ['West Lampeter'], + 'region_a': ['PA'], + 'region': ['Pennsylvania'], + 'locality': ['Lampeter'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Wheatland Mills'] }, 'name': { 'default': 'Lampeter-Strasburg High School' @@ -87,14 +87,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'localadmin': 'East Lampeter', - 'region_a': 'PA', - 'region': 'Pennsylvania', - 'locality': 'Smoketown', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Greenland' + 'localadmin': ['East Lampeter'], + 'region_a': ['PA'], + 'region': ['Pennsylvania'], + 'locality': ['Smoketown'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Greenland'] }, 'name': { 'default': 'East Lampeter High School' @@ -114,14 +114,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'region': 'Pennsylvania', - 'locality': 'Lampeter', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Wheatland Mills', - 'localadmin': 'West Lampeter', - 'region_a': 'PA' + 'region': ['Pennsylvania'], + 'locality': ['Lampeter'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Wheatland Mills'], + 'localadmin': ['West Lampeter'], + 'region_a': ['PA'] }, 'name': { 'default': 'Lampeter-Strasburg High School' @@ -141,14 +141,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'region': 'Pennsylvania', - 'locality': 'Smoketown', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Greenland', - 'localadmin': 'East Lampeter', - 'region_a': 'PA' + 'region': ['Pennsylvania'], + 'locality': ['Smoketown'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Greenland'], + 'localadmin': ['East Lampeter'], + 'region_a': ['PA'] }, 'name': { 'default': 'East Lampeter School' @@ -168,14 +168,14 @@ module.exports = [ }, 'address': {}, 'parent': { - 'region': 'Pennsylvania', - 'locality': 'Smoketown', - 'country_a': 'USA', - 'county': 'Lancaster County', - 'country': 'United States', - 'neighbourhood': 'Greenland', - 'localadmin': 'East Lampeter', - 'region_a': 'PA' + 'region': ['Pennsylvania'], + 'locality': ['Smoketown'], + 'country_a': ['USA'], + 'county': ['Lancaster County'], + 'country': ['United States'], + 'neighbourhood': ['Greenland'], + 'localadmin': ['East Lampeter'], + 'region_a': ['PA'] }, 'name': { 'default': 'East Lampeter School' @@ -195,13 +195,13 @@ module.exports = [ }, 'address': {}, 'parent': { - 'region': 'Missouri', - 'locality': 'Strasburg', - 'country_a': 'USA', - 'county': 'Cass County', - 'country': 'United States', - 'localadmin': 'Polk', - 'region_a': 'MO' + 'region': ['Missouri'], + 'locality': ['Strasburg'], + 'country_a': ['USA'], + 'county': ['Cass County'], + 'country': ['United States'], + 'localadmin': ['Polk'], + 'region_a': ['MO'] }, 'name': { 'default': 'Strasburg School' @@ -224,13 +224,13 @@ module.exports = [ 'default': 'Strasburg High School' }, 'parent': { - 'region_a': 'VA', - 'region': 'Virginia', - 'locality': 'Strasburg', - 'country_a': 'USA', - 'county': 'Shenandoah County', - 'country': 'United States', - 'neighbourhood': 'Strasburg Junction' + 'region_a': ['VA'], + 'region': ['Virginia'], + 'locality': ['Strasburg'], + 'country_a': ['USA'], + 'county': ['Shenandoah County'], + 'country': ['United States'], + 'neighbourhood': ['Strasburg Junction'] }, 'category': [ 'education' @@ -250,13 +250,13 @@ module.exports = [ 'default': 'Strasburg High School' }, 'parent': { - 'localadmin': 'Strasburg', - 'region_a': 'ND', - 'region': 'North Dakota', - 'locality': 'Strasburg', - 'country_a': 'USA', - 'county': 'Emmons County', - 'country': 'United States' + 'localadmin': ['Strasburg'], + 'region_a': ['ND'], + 'region': ['North Dakota'], + 'locality': ['Strasburg'], + 'country_a': ['USA'], + 'county': ['Emmons County'], + 'country': ['United States'] }, 'category': [ 'education' @@ -276,13 +276,13 @@ module.exports = [ 'default': 'Strasburg High School' }, 'parent': { - 'localadmin': 'Franklin', - 'region_a': 'OH', - 'region': 'Ohio', - 'locality': 'Strasburg', - 'country_a': 'USA', - 'county': 'Tuscarawas County', - 'country': 'United States' + 'localadmin': ['Franklin'], + 'region_a': ['OH'], + 'region': ['Ohio'], + 'locality': ['Strasburg'], + 'country_a': ['USA'], + 'county': ['Tuscarawas County'], + 'country': ['United States'] }, 'category': [ 'education' diff --git a/test/unit/middleware/confidenceScore.js b/test/unit/middleware/confidenceScore.js index 7d516f16..f629874b 100644 --- a/test/unit/middleware/confidenceScore.js +++ b/test/unit/middleware/confidenceScore.js @@ -71,9 +71,9 @@ module.exports.tests.confidenceScore = function(test, common) { center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, parent: { - country: 'country1', - region: 'state1', - county: 'city1' + country: ['country1'], + region: ['state1'], + county: ['city1'] } }, { _score: 20, @@ -81,9 +81,9 @@ module.exports.tests.confidenceScore = function(test, common) { center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, parent: { - country: 'country2', - region: 'state2', - county: 'city2' + country: ['country2'], + region: ['state2'], + county: ['city2'] } }], meta: {scores: [10]} diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index da904e63..d8714af5 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -16,9 +16,9 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { 'street': 'Main St' }, 'parent': { - 'region': 'Dungannon', - 'country_a': 'GBR', - 'country': 'United Kingdom' + 'region': ['Dungannon'], + 'country_a': ['GBR'], + 'country': ['United Kingdom'] } }; @@ -33,12 +33,12 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { 'street': 'Grolmanstraße' }, 'parent': { - 'region': 'Berlin', - 'locality': 'Berlin', - 'country_a': 'DEU', - 'county': 'Berlin', - 'country': 'Germany', - 'neighbourhood': 'Hansaviertel' + 'region': ['Berlin'], + 'locality': ['Berlin'], + 'country_a': ['DEU'], + 'county': ['Berlin'], + 'country': ['Germany'], + 'neighbourhood': ['Hansaviertel'] } }; diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js index ea20c4d0..84147783 100644 --- a/test/unit/mock/backend.js +++ b/test/unit/mock/backend.js @@ -15,7 +15,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){ value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - parent: { country: 'country1', region: 'state1', county: 'city1' } + parent: { country: ['country1'], region: ['state1'], county: ['city1'] } } }, { _id: 'myid2', @@ -25,7 +25,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){ value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - parent: { country: 'country2', region: 'state2', county: 'city2' } + parent: { country: ['country2'], region: ['state2'], county: ['city2'] } } }])); }; @@ -43,7 +43,7 @@ responses['client/mget/ok/1'] = function( cmd, cb ){ value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - parent: { country: 'country1', region: 'state1', county: 'city1' } + parent: { country: ['country1'], region: ['state1'], county: ['city1'] } } }, { _id: 'myid2', @@ -54,7 +54,7 @@ responses['client/mget/ok/1'] = function( cmd, cb ){ value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - parent: { country: 'country2', region: 'state2', county: 'city2' } + parent: { country: ['country2'], region: ['state2'], county: ['city2'] } } }])); }; diff --git a/test/unit/service/mget.js b/test/unit/service/mget.js index dd093c9f..e2723e6e 100644 --- a/test/unit/service/mget.js +++ b/test/unit/service/mget.js @@ -20,14 +20,14 @@ module.exports.tests.functional_success = function(test, common) { value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - parent: { country: 'country1', region: 'state1', county: 'city1' } + parent: { country: ['country1'], region: ['state1'], county: ['city1'] } }, { _id: 'myid2', _type: 'mytype2', value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - parent: { country: 'country2', region: 'state2', county: 'city2' } + parent: { country: ['country2'], region: ['state2'], county: ['city2'] } } ]; diff --git a/test/unit/service/search.js b/test/unit/service/search.js index 37eb93af..0de8e0fa 100644 --- a/test/unit/service/search.js +++ b/test/unit/service/search.js @@ -23,7 +23,7 @@ module.exports.tests.functional_success = function(test, common) { value: 1, center_point: { lat: 100.1, lon: -50.5 }, name: { default: 'test name1' }, - parent: { country: 'country1', region: 'state1', county: 'city1' } + parent: { country: ['country1'], region: ['state1'], county: ['city1'] } }, { _id: 'myid2', _type: 'mytype2', @@ -31,7 +31,7 @@ module.exports.tests.functional_success = function(test, common) { value: 2, center_point: { lat: 100.2, lon: -51.5 }, name: { default: 'test name2' }, - parent: { country: 'country2', region: 'state2', county: 'city2' } + parent: { country: ['country2'], region: ['state2'], county: ['city2'] } } ]; From a4f6e70d255cb5cd16cb2b15a25c9d01e3fdb296 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 11 Mar 2016 18:47:45 +0100 Subject: [PATCH 074/106] upgrade to pelias-query@6.1.0 --- package.json | 2 +- .../autocomplete_linguistic_final_token.js | 16 ++++----------- .../fixture/autocomplete_linguistic_focus.js | 16 ++++----------- ...tocomplete_linguistic_focus_null_island.js | 16 ++++----------- ...autocomplete_linguistic_multiple_tokens.js | 16 ++++----------- .../fixture/autocomplete_linguistic_only.js | 16 ++++----------- .../autocomplete_linguistic_with_admin.js | 18 +++++------------ test/unit/fixture/search_boundary_country.js | 16 ++++----------- test/unit/fixture/search_full_address.js | 16 ++++----------- test/unit/fixture/search_linguistic_bbox.js | 16 ++++----------- test/unit/fixture/search_linguistic_focus.js | 16 ++++----------- .../fixture/search_linguistic_focus_bbox.js | 16 ++++----------- .../search_linguistic_focus_null_island.js | 16 ++++----------- test/unit/fixture/search_linguistic_only.js | 16 ++++----------- .../fixture/search_linguistic_viewport.js | 20 ++++++------------- ...search_linguistic_viewport_min_diagonal.js | 20 ++++++------------- test/unit/fixture/search_partial_address.js | 16 ++++----------- test/unit/fixture/search_regions_address.js | 16 ++++----------- 18 files changed, 74 insertions(+), 210 deletions(-) diff --git a/package.json b/package.json index d44b0576..919c19c5 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", - "pelias-query": "^6.0.0", + "pelias-query": "^6.1.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", "through2": "2.0.1" diff --git a/test/unit/fixture/autocomplete_linguistic_final_token.js b/test/unit/fixture/autocomplete_linguistic_final_token.js index 1641ac44..fbe80052 100644 --- a/test/unit/fixture/autocomplete_linguistic_final_token.js +++ b/test/unit/fixture/autocomplete_linguistic_final_token.js @@ -31,15 +31,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -60,15 +56,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 513136b9..4f722b84 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -75,15 +75,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -104,15 +100,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 3985d061..d9c04fd1 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -75,15 +75,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -104,15 +100,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js index 16a25605..9018fdab 100644 --- a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js +++ b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js @@ -42,15 +42,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -71,15 +67,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 05ff6c43..24b89ad9 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -31,15 +31,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -60,15 +56,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_with_admin.js b/test/unit/fixture/autocomplete_linguistic_with_admin.js index 2c27f670..7b2d78c3 100644 --- a/test/unit/fixture/autocomplete_linguistic_with_admin.js +++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js @@ -99,18 +99,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } } }, { @@ -131,18 +127,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index 5dde3344..4bf45315 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -49,15 +49,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -78,15 +74,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 859cfda4..8e9fbaa8 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js index c9859aeb..5bb5907c 100644 --- a/test/unit/fixture/search_linguistic_bbox.js +++ b/test/unit/fixture/search_linguistic_bbox.js @@ -39,15 +39,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -68,15 +64,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js index 562f32c3..5d03d66d 100644 --- a/test/unit/fixture/search_linguistic_focus.js +++ b/test/unit/fixture/search_linguistic_focus.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js index bd7c40ab..96fe92f6 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox.js +++ b/test/unit/fixture/search_linguistic_focus_bbox.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js index cf8b1625..0924475d 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island.js +++ b/test/unit/fixture/search_linguistic_focus_null_island.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js index 53ece28e..58c05826 100644 --- a/test/unit/fixture/search_linguistic_only.js +++ b/test/unit/fixture/search_linguistic_only.js @@ -39,15 +39,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -68,15 +64,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js index 7cf6d138..be76ab05 100644 --- a/test/unit/fixture/search_linguistic_viewport.js +++ b/test/unit/fixture/search_linguistic_viewport.js @@ -77,18 +77,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } + 'boost_mode': 'replace' } }, { @@ -109,18 +105,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js index c5306919..cf44d0d8 100644 --- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js +++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js @@ -77,18 +77,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } + 'boost_mode': 'replace' } }, { @@ -109,18 +105,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index d857e31b..6c4174b6 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index 25beb280..0c2ddd8f 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] From 6f7b2ac68a3f65e1a1f2fac98cd2610874bab312 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 10 Mar 2016 20:03:06 -0500 Subject: [PATCH 075/106] Pass esclient config to elasticsearch module This fixes a regression in #423 that breaks our configuration in dev and production. --- src/backend.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend.js b/src/backend.js index 3d2bc993..c983dbba 100644 --- a/src/backend.js +++ b/src/backend.js @@ -1,6 +1,6 @@ - +var config = require( 'pelias-config' ).generate().esclient; var Backend = require('geopipes-elasticsearch-backend'), - client = require('elasticsearch').Client(), + client = require('elasticsearch').Client(config), backends = {}; function getBackend( index, type ){ From b4fb62f2c9d546da669f83e81f517b021439300a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 Mar 2016 15:13:41 -0500 Subject: [PATCH 076/106] Test that src/backend passes correct config to elasticsearch --- test/unit/run.js | 1 + test/unit/src/backend.js | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 test/unit/src/backend.js diff --git a/test/unit/run.js b/test/unit/run.js index de13bbe0..ccb3c790 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -45,6 +45,7 @@ var tests = [ require('./sanitiser/_sources'), require('./sanitiser/_sources_and_layers'), require('./sanitiser/_text'), + require('./src/backend'), require('./sanitiser/autocomplete'), require('./sanitiser/place'), require('./sanitiser/reverse'), diff --git a/test/unit/src/backend.js b/test/unit/src/backend.js new file mode 100644 index 00000000..1b750396 --- /dev/null +++ b/test/unit/src/backend.js @@ -0,0 +1,40 @@ +var proxyquire = require('proxyquire'); + +var stubConfig = { + generate: function generate() { + return { + esclient: { + hosts: [ + 'http://notLocalhost:9200', + 'http://anotherHost:9200' + ] + } + }; + } +}; + + +module.exports.tests = {}; + +module.exports.tests.config_properly_passed = function(test, common) { + test('Elasticsearch config is properly passed to elasticsearch module', function(t) { + var stubElasticsearchClient = { + Client: function(config) { + t.deepEquals(config.hosts, [ 'http://notLocalhost:9200', 'http://anotherHost:9200' ], 'hosts set correctly' ); + t.end(); + } + }; + + proxyquire('../../../src/backend', { 'pelias-config': stubConfig, 'elasticsearch': stubElasticsearchClient } ); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANTIZE src/backend ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; From ab5004273b1dd761fcc7f7a2d17de6d316b6c97f Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Fri, 11 Mar 2016 17:09:12 -0500 Subject: [PATCH 077/106] Fix unit tests due to new version of pelias-query --- .../autocomplete_linguistic_final_token.js | 16 ++++----------- .../fixture/autocomplete_linguistic_focus.js | 16 ++++----------- ...tocomplete_linguistic_focus_null_island.js | 16 ++++----------- ...autocomplete_linguistic_multiple_tokens.js | 16 ++++----------- .../fixture/autocomplete_linguistic_only.js | 16 ++++----------- .../autocomplete_linguistic_with_admin.js | 20 ++++++------------- test/unit/fixture/search_boundary_country.js | 16 ++++----------- test/unit/fixture/search_full_address.js | 16 ++++----------- test/unit/fixture/search_linguistic_bbox.js | 16 ++++----------- test/unit/fixture/search_linguistic_focus.js | 16 ++++----------- .../fixture/search_linguistic_focus_bbox.js | 16 ++++----------- .../search_linguistic_focus_null_island.js | 16 ++++----------- test/unit/fixture/search_linguistic_only.js | 16 ++++----------- .../fixture/search_linguistic_viewport.js | 20 ++++++------------- ...search_linguistic_viewport_min_diagonal.js | 20 ++++++------------- test/unit/fixture/search_partial_address.js | 16 ++++----------- test/unit/fixture/search_regions_address.js | 16 ++++----------- 17 files changed, 74 insertions(+), 210 deletions(-) diff --git a/test/unit/fixture/autocomplete_linguistic_final_token.js b/test/unit/fixture/autocomplete_linguistic_final_token.js index 1641ac44..fbe80052 100644 --- a/test/unit/fixture/autocomplete_linguistic_final_token.js +++ b/test/unit/fixture/autocomplete_linguistic_final_token.js @@ -31,15 +31,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -60,15 +56,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 513136b9..4f722b84 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -75,15 +75,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -104,15 +100,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 3985d061..d9c04fd1 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -75,15 +75,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -104,15 +100,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js index 16a25605..9018fdab 100644 --- a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js +++ b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js @@ -42,15 +42,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -71,15 +67,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 05ff6c43..24b89ad9 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -31,15 +31,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -60,15 +56,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 }] diff --git a/test/unit/fixture/autocomplete_linguistic_with_admin.js b/test/unit/fixture/autocomplete_linguistic_with_admin.js index 2c27f670..245f6451 100644 --- a/test/unit/fixture/autocomplete_linguistic_with_admin.js +++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js @@ -99,18 +99,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } + 'boost_mode': 'replace' } }, { @@ -131,18 +127,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 3 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index 5dde3344..4bf45315 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -49,15 +49,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -78,15 +74,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 859cfda4..8e9fbaa8 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js index c9859aeb..5bb5907c 100644 --- a/test/unit/fixture/search_linguistic_bbox.js +++ b/test/unit/fixture/search_linguistic_bbox.js @@ -39,15 +39,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -68,15 +64,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js index 562f32c3..5d03d66d 100644 --- a/test/unit/fixture/search_linguistic_focus.js +++ b/test/unit/fixture/search_linguistic_focus.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js index bd7c40ab..96fe92f6 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox.js +++ b/test/unit/fixture/search_linguistic_focus_bbox.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js index cf8b1625..0924475d 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island.js +++ b/test/unit/fixture/search_linguistic_focus_null_island.js @@ -69,15 +69,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -98,15 +94,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js index 53ece28e..58c05826 100644 --- a/test/unit/fixture/search_linguistic_only.js +++ b/test/unit/fixture/search_linguistic_only.js @@ -39,15 +39,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -68,15 +64,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js index 7cf6d138..be76ab05 100644 --- a/test/unit/fixture/search_linguistic_viewport.js +++ b/test/unit/fixture/search_linguistic_viewport.js @@ -77,18 +77,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } + 'boost_mode': 'replace' } }, { @@ -109,18 +105,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js index c5306919..cf44d0d8 100644 --- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js +++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js @@ -77,18 +77,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - } + 'boost_mode': 'replace' } }, { @@ -109,18 +105,14 @@ module.exports = { { 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 } ], 'score_mode': 'first', - 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - } + 'boost_mode': 'replace' } } ] diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index d857e31b..6c4174b6 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index 25beb280..0c2ddd8f 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -41,15 +41,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'popularity' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'popularity' + 'field': 'popularity', + 'missing': 1 }, 'weight': 1 }] @@ -70,15 +66,11 @@ module.exports = { 'max_boost': 20, 'score_mode': 'first', 'boost_mode': 'replace', - 'filter': { - 'exists': { - 'field': 'population' - } - }, 'functions': [{ 'field_value_factor': { 'modifier': 'log1p', - 'field': 'population' + 'field': 'population', + 'missing': 1 }, 'weight': 2 }] From 766b68ef7c9d88583bdfb8c4e631b40457fed50c Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Mon, 14 Mar 2016 15:27:04 +0200 Subject: [PATCH 078/106] Make street/number flipping configurable pelias_config.api.localization.flipNumberAndStreetCountries can define an array of ISO 3166-1 alpha-3 country codes, for which the street number should come after street name. --- middleware/localNamingConventions.js | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 6170cda2..bd158639 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -1,5 +1,20 @@ +var check = require('check-types'); + +var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL']; function setup() { + var api = require('pelias-config').generate().api; + var settings = api.localization; + if (settings && settings.flipNumberAndStreetCountries) { + var countries = settings.flipNumberAndStreetCountries; + for (var i=0; i 'Grolmanstraße 101' function flipNumberAndStreet(place) { var standard = ( place.address.number + ' ' + place.address.street ), From 4ed5296d79c34e32cbe22073599234832a69e180 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:04:54 -0400 Subject: [PATCH 079/106] Update middleware with new prop name --- middleware/confidenceScore.js | 25 +++++++++++++------------ middleware/dedupe.js | 10 +++++----- middleware/localNamingConventions.js | 10 +++++----- middleware/renamePlacenames.js | 4 ++-- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index a0382b25..df48b2b1 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -100,9 +100,10 @@ function checkForDealBreakers(req, hit) { return true; } - if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address) && - req.clean.parsed_text.postalcode !== hit.address.zip) { - logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')'); + if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) && + req.clean.parsed_text.postalcode !== hit.address_parts.zip) { + logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + + ' !== ' + hit.address_parts.zip + ')'); return true; } } @@ -155,8 +156,8 @@ function checkName(text, parsed_text, hit) { */ function checkQueryType(text, hit) { if (check.assigned(text) && check.assigned(text.number) && - (check.undefined(hit.address) || - (check.assigned(hit.address) && check.undefined(hit.address.number)))) { + (check.undefined(hit.address_parts) || + (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) { return 0; } return 1; @@ -206,10 +207,10 @@ function propMatch(textProp, hitProp, expectEnriched) { * @param {string} [text.state] * @param {string} [text.country] * @param {object} hit - * @param {object} [hit.address] - * @param {string|number} [hit.address.number] - * @param {string} [hit.address.street] - * @param {string|number} [hit.address.zip] + * @param {object} [hit.address_parts] + * @param {string|number} [hit.address_parts.number] + * @param {string} [hit.address_parts.street] + * @param {string|number} [hit.address_parts.zip] * @param {Array} [hit.parent.region_a] * @param {Array} [hit.parent.country_a] * @returns {number} @@ -219,9 +220,9 @@ function checkAddress(text, hit) { var res = 0; if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) { - res += propMatch(text.number, (hit.address ? hit.address.number : null), false); - res += propMatch(text.street, (hit.address ? hit.address.street : null), false); - res += propMatch(text.postalcode, (hit.address ? hit.address.zip: null), true); + res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false); + res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false); + res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true); res += propMatch(text.state, hit.parent.region_a[0], true); res += propMatch(text.country, hit.parent.country_a[0], true); diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 9e0e5c6a..d9336ffe 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -54,12 +54,12 @@ function isDifferent(item1, item2) { propMatch(item1, item2, 'name'); } - if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) { - propMatch(item1.address, item2.address, 'number'); - propMatch(item1.address, item2.address, 'street'); - propMatch(item1.address, item2.address, 'zip'); + if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) { + propMatch(item1.address_parts, item2.address_parts, 'number'); + propMatch(item1.address_parts, item2.address_parts, 'street'); + propMatch(item1.address_parts, item2.address_parts, 'zip'); } - else if (item1.address !== item2.address) { + else if (item1.address_parts !== item2.address_parts) { throw new Error('different'); } } diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 6170cda2..c78288ce 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -14,9 +14,9 @@ function applyLocalNamingConventions(req, res, next) { res.data.filter(function(place){ // only relevant for German addresses if( place.parent.country_a.indexOf('DEU') === -1 ){ return false; } - if( !place.hasOwnProperty('address') ){ return false; } - if( !place.address.hasOwnProperty('number') ){ return false; } - if( !place.address.hasOwnProperty('street') ){ return false; } + if( !place.hasOwnProperty('address_parts') ){ return false; } + if( !place.address_parts.hasOwnProperty('number') ){ return false; } + if( !place.address_parts.hasOwnProperty('street') ){ return false; } return true; }) .forEach( flipNumberAndStreet ); @@ -27,8 +27,8 @@ function applyLocalNamingConventions(req, res, next) { // DE address should have the housenumber and street name flipped // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101' function flipNumberAndStreet(place) { - var standard = ( place.address.number + ' ' + place.address.street ), - flipped = ( place.address.street + ' ' + place.address.number ); + var standard = ( place.address_parts.number + ' ' + place.address_parts.street ), + flipped = ( place.address_parts.street + ' ' + place.address_parts.number ); // flip street name and housenumber if( place.name.default === standard ){ diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 8f171e24..5bb7df67 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -58,9 +58,9 @@ function renamePlacenames(req, res, next) { * Rename the fields in one record */ function renameOneRecord(place) { - if (place.address) { + if (place.address_parts) { Object.keys(ADDRESS_PROPS).forEach(function (prop) { - place[ADDRESS_PROPS[prop]] = place.address[prop]; + place[ADDRESS_PROPS[prop]] = place.address_parts[prop]; }); } From 0917a84988be59c5281c4411aa462f05155a4759 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:05:25 -0400 Subject: [PATCH 080/106] Update property names in query defaults --- query/autocomplete_defaults.js | 6 +++--- query/reverse_defaults.js | 6 +++--- query/search_defaults.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 7bb430cc..ba52a049 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'multiply', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 2000, 'admin:country_a:analyzer': 'standard', diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index c32b1c3d..306efaac 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'replace', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 3, 'admin:country_a:analyzer': 'standard', diff --git a/query/search_defaults.js b/query/search_defaults.js index e9a918cd..ea0dc87f 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -39,15 +39,15 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:boost_mode': 'replace', 'address:housenumber:analyzer': 'peliasHousenumber', - 'address:housenumber:field': 'address.number', + 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, 'address:street:analyzer': 'peliasStreet', - 'address:street:field': 'address.street', + 'address:street:field': 'address_parts.street', 'address:street:boost': 5, 'address:postcode:analyzer': 'peliasZip', - 'address:postcode:field': 'address.zip', + 'address:postcode:field': 'address_parts.zip', 'address:postcode:boost': 20, 'admin:country_a:analyzer': 'standard', From 74467e91e636e02a20aa6fc7d1853eac0bd09132 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:05:57 -0400 Subject: [PATCH 081/106] Update tests and fixtures --- .../dedupe_elasticsearch_nonascii_results.js | 2 +- .../fixture/dedupe_elasticsearch_results.js | 22 +++++++++---------- test/unit/fixture/search_full_address.js | 6 ++--- test/unit/fixture/search_regions_address.js | 4 ++-- .../unit/middleware/localNamingConventions.js | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js index b2a6c383..437060b9 100644 --- a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js @@ -55,7 +55,7 @@ module.exports = [ 'name': { 'default': '万里加油站' }, - 'address': { + 'address_parts': { 'street': 'S308', 'postalcode': '312044' }, diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index a57a3a7f..3fb6f3f9 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -4,7 +4,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.039265 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -31,7 +31,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.039265 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -58,7 +58,7 @@ module.exports = [ 'lon': -76.23246, 'lat': 39.99288 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['West Lampeter'], 'region_a': ['PA'], @@ -85,7 +85,7 @@ module.exports = [ 'lon': -76.20746, 'lat': 40.03927 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'localadmin': ['East Lampeter'], 'region_a': ['PA'], @@ -112,7 +112,7 @@ module.exports = [ 'lon': -76.232457, 'lat': 39.992877 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Lampeter'], @@ -139,7 +139,7 @@ module.exports = [ 'lon': -76.207456, 'lat': 40.038987 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Smoketown'], @@ -166,7 +166,7 @@ module.exports = [ 'lon': -76.20746, 'lat': 40.03899 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Pennsylvania'], 'locality': ['Smoketown'], @@ -193,7 +193,7 @@ module.exports = [ 'lon': -94.167445, 'lat': 38.762788 }, - 'address': {}, + 'address_parts': {}, 'parent': { 'region': ['Missouri'], 'locality': ['Strasburg'], @@ -219,7 +219,7 @@ module.exports = [ 'lon': -78.36317, 'lat': 38.98445 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, @@ -245,7 +245,7 @@ module.exports = [ 'lon': -100.16516, 'lat': 46.13427 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, @@ -271,7 +271,7 @@ module.exports = [ 'lon': -81.532392, 'lat': 40.597578 }, - 'address': {}, + 'address_parts': {}, 'name': { 'default': 'Strasburg High School' }, diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 8e9fbaa8..8a8290ab 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -77,7 +77,7 @@ module.exports = { } },{ 'match': { - 'address.number': { + 'address_parts.number': { 'query': '123', 'boost': vs['address:housenumber:boost'], 'analyzer': vs['address:housenumber:analyzer'] @@ -85,7 +85,7 @@ module.exports = { } }, { 'match': { - 'address.street': { + 'address_parts.street': { 'query': 'main st', 'boost': vs['address:street:boost'], 'analyzer': vs['address:street:analyzer'] @@ -93,7 +93,7 @@ module.exports = { } }, { 'match': { - 'address.zip': { + 'address_parts.zip': { 'query': '10010', 'boost': vs['address:postcode:boost'], 'analyzer': vs['address:postcode:analyzer'] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index 0c2ddd8f..e0c05f3c 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -77,7 +77,7 @@ module.exports = { } },{ 'match': { - 'address.number': { + 'address_parts.number': { 'query': '1', 'boost': vs['address:housenumber:boost'], 'analyzer': vs['address:housenumber:analyzer'] @@ -85,7 +85,7 @@ module.exports = { } }, { 'match': { - 'address.street': { + 'address_parts.street': { 'query': 'water st', 'boost': vs['address:street:boost'], 'analyzer': vs['address:street:analyzer'] diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index d8714af5..d22642db 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -10,7 +10,7 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { '_type': 'test', 'name': { 'default': '1 Main St' }, 'center_point': { 'lon': -7.131521, 'lat': 54.428866 }, - 'address': { + 'address_parts': { 'zip': 'BT77 0BG', 'number': '1', 'street': 'Main St' @@ -27,7 +27,7 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { '_type': 'test', 'name': { 'default': '23 Grolmanstraße' }, 'center_point': { 'lon': 13.321487, 'lat': 52.506781 }, - 'address': { + 'address_parts': { 'zip': '10623', 'number': '23', 'street': 'Grolmanstraße' From 582fd50902b251f28988dfd8779bdb62adb35faa Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 15 Mar 2016 16:06:23 -0400 Subject: [PATCH 082/106] Update pelias-query package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 919c19c5..49e00498 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", - "pelias-query": "^6.1.0", + "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", "through2": "2.0.1" From 6ab44e0aa21a544e3a091a2b42668aea1f3e3ccc Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Wed, 16 Mar 2016 10:14:20 +0200 Subject: [PATCH 083/106] neighborhood -> neighbourhood --- middleware/dedupe.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 5f65e54b..50557a49 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -43,7 +43,7 @@ function isDifferent(item1, item2) { propMatch(item1.parent, item2.parent, 'region_a'); propMatch(item1.parent, item2.parent, 'country'); propMatch(item1.parent, item2.parent, 'locality'); - propMatch(item1.parent, item2.parent, 'neighborhood'); + propMatch(item1.parent, item2.parent, 'neighbourhood'); } else if (item1.parent !== item2.parent) { throw new Error('different'); From 0064f6f2570b34ab9b56fb2043e06816ee933c5f Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Wed, 16 Mar 2016 10:15:30 +0200 Subject: [PATCH 084/106] Add two new test entries to test neighbourhood and locality in deduping --- .../fixture/dedupe_elasticsearch_results.js | 54 +++++++++++++++++++ test/unit/middleware/dedupe.js | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index 84b279d3..ca623e4e 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -26,6 +26,60 @@ module.exports = [ '_score': 1.2367082, 'confidence': 0.879 }, + { // same as above, but change the neighbourhood + 'center_point': { + 'lon': -77.207456, + 'lat': 41.039265 + }, + 'address': {}, + 'parent': { + 'localadmin': 'East Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Smoketown', + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Blueland' // ### + }, + 'name': { + 'default': 'East Lampeter High School' + }, + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'venue', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { // same as #1, but change the locality + 'center_point': { + 'lon': -73.207456, + 'lat': 42.039265 + }, + 'address': {}, + 'parent': { + 'localadmin': 'East Lampeter', + 'region_a': 'PA', + 'region': 'Pennsylvania', + 'locality': 'Firetown', // ### + 'country_a': 'USA', + 'county': 'Lancaster County', + 'country': 'United States', + 'neighbourhood': 'Greenland' + }, + 'name': { + 'default': 'East Lampeter High School' + }, + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'venue', + '_score': 1.2367082, + 'confidence': 0.879 + }, { 'center_point': { 'lon': -76.207456, diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js index 6707d1f9..ad553f9c 100644 --- a/test/unit/middleware/dedupe.js +++ b/test/unit/middleware/dedupe.js @@ -16,7 +16,7 @@ module.exports.tests.dedupe = function(test, common) { data: data }; - var expectedCount = 7; + var expectedCount = 9; dedupe(req, res, function () { t.equal(res.data.length, expectedCount, 'results have fewer items than before'); t.end(); From 00a4bd52a362f952b3fd01bf766ed357c55b15e2 Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Wed, 16 Mar 2016 11:35:50 +0200 Subject: [PATCH 085/106] Bugfix: deduping caused an error if an array property was missing Conversion from array to string should happen independently for the compared properties, not only when both are arrays. --- middleware/dedupe.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 50557a49..d130eb7b 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -91,10 +91,8 @@ function propMatch(item1, item2, prop) { // in the case the property is an array (currently only in parent schema) // simply take the 1st item. this will change in the near future to support multiple hierarchies - if (_.isArray(prop1) && _.isArray(prop2)) { - prop1 = prop1[0]; - prop2= prop2[0]; - } + if (_.isArray(prop1)) { prop1 = prop1[0]; } + if (_.isArray(prop2)) { prop2 = prop2[0]; } if (normalizeString(prop1) !== normalizeString(prop2)) { throw new Error('different'); From 34434ad029bae90c20e6282eb54291a582e91400 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 16 Mar 2016 12:52:49 -0400 Subject: [PATCH 086/106] Update travis versions Consolidate on the latest minor versions of Node 4 and 5 --- .travis.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f042d3c2..0abfae12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,18 +3,12 @@ language: node_js node_js: - 0.10 - 0.12 - - 4.0 - - 4.1 - - 4.2 - - 4.3 - - 5.6 + - 4.4 + - 5.8 matrix: allow_failures: - - node_js: 4.0 - - node_js: 4.1 - - node_js: 4.2 - - node_js: 4.3 - - node_js: 5.6 + - node_js: 4.4 + - node_js: 5.8 env: global: - CXX=g++-4.8 From 0d0651f8ac3b77d84112452c0bdc9541e7f56c91 Mon Sep 17 00:00:00 2001 From: Vesa Meskanen Date: Thu, 17 Mar 2016 09:05:32 +0200 Subject: [PATCH 087/106] Get rid of ugly for-loops using lodash --- middleware/localNamingConventions.js | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index ec416790..dd026bb3 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -1,4 +1,5 @@ var check = require('check-types'); +var _ = require('lodash'); var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL']; @@ -7,12 +8,7 @@ function setup() { var settings = api.localization; if (settings && settings.flipNumberAndStreetCountries) { var countries = settings.flipNumberAndStreetCountries; - for (var i=0; i Date: Thu, 17 Mar 2016 10:59:20 +0200 Subject: [PATCH 088/106] Unit test for configured flip street/number countries --- .../unit/middleware/localNamingConventions.js | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index d22642db..c544abe7 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -1,4 +1,19 @@ -var localNamingConventions = require('../../../middleware/localNamingConventions'); + +var proxyquire = require('proxyquire'); + +var customConfig = { + generate: function generate() { + return { + api : { + localization : { // expand the set of flipped countries + flipNumberAndStreetCountries : ['NLD'] // Netherlands + } + } + }; + } +}; + +var localNamingConventions = proxyquire('../../../middleware/localNamingConventions', { 'pelias-config': customConfig }); module.exports.tests = {}; @@ -42,8 +57,28 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { } }; + var nlAddress = { + '_id': 'test3', + '_type': 'test', + 'name': { 'default': '117 Keizersgracht' }, + 'center_point': { 'lon': 4.887545, 'lat': 52.376795 }, + 'address_parts': { + 'zip': '1015', + 'number': '117', + 'street': 'Keizersgracht' + }, + 'parent': { + 'region': ['Amsterdam'], + 'locality': ['Amsterdam'], + 'country_a': ['NLD'], + 'county': ['Noord-Holland'], + 'country': ['Netherlands'], + 'neighbourhood': ['Grachtengordel-West'] + } + }; + var req = {}, - res = { data: [ ukAddress, deAddress ] }, + res = { data: [ ukAddress, deAddress, nlAddress ] }, middleware = localNamingConventions(); test('flipNumberAndStreet', function(t) { @@ -57,6 +92,10 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101' t.equal( res.data[1].name.default, 'Grolmanstraße 23', 'flipped name' ); + // NLD address should have the housenumber and street name flipped, too + // this definition comes from pelias configuration + t.equal( res.data[2].name.default, 'Keizersgracht 117', 'flipped name' ); + t.end(); }); }); From 77f396d5cb5ab9f7981400885b2c959d2ccc0381 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 18 Mar 2016 09:33:24 -0400 Subject: [PATCH 089/106] exposed `macrocounty` and `macroregion` fields in geojsonify --- helper/geojsonify.js | 6 ++++++ test/unit/helper/geojsonify.js | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 8910767d..1f9a60a8 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -16,9 +16,15 @@ var DETAILS_PROPS = [ 'country', 'country_id', 'country_a', + 'macroregion', + 'macroregion_id', + 'macroregion_a', 'region', 'region_id', 'region_a', + 'macrocounty', + 'macrocounty_id', + 'macrocounty_a', 'county', 'county_id', 'county_a', diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 0aa6b39c..7b371b27 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -246,6 +246,15 @@ module.exports.tests.search = function(test, common) { 'country_a': [ 'USA' ], + 'macroregion': [ + 'MacroRegion Name' + ], + 'macroregion_id': [ + 'MacroRegion Id' + ], + 'macroregion_a': [ + 'MacroRegion Abbreviation' + ], 'region': [ 'New York' ], @@ -255,6 +264,15 @@ module.exports.tests.search = function(test, common) { 'region_a': [ 'NY' ], + 'macrocounty': [ + 'MacroCounty Name' + ], + 'macrocounty_id': [ + 'MacroCounty Id' + ], + 'macrocounty_a': [ + 'MacroCounty Abbreviation' + ], 'county': [ 'Kings County' ], @@ -300,9 +318,15 @@ module.exports.tests.search = function(test, common) { 'country': 'United States', 'country_id': '85633793', 'country_a': 'USA', + 'macroregion': 'MacroRegion Name', + 'macroregion_id': 'MacroRegion Id', + 'macroregion_a': 'MacroRegion Abbreviation', 'region': 'New York', 'region_id': '85688543', 'region_a': 'NY', + 'macrocounty': 'MacroCounty Name', + 'macrocounty_id': 'MacroCounty Id', + 'macrocounty_a': 'MacroCounty Abbreviation', 'county': 'Kings County', 'county_id': '102082361', 'localadmin': 'Brooklyn', From 1204727f74376d3e8eb23de6417ff1a4f84c5d1d Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 18 Mar 2016 11:11:16 -0400 Subject: [PATCH 090/106] added `macrocounty` and `macroregion` to properties to pull from parent --- middleware/renamePlacenames.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 5bb7df67..152ad404 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -22,9 +22,15 @@ var PARENT_PROPS = [ 'country', 'country_id', 'country_a', + 'macroregion', + 'macroregion_id', + 'macroregion_a', 'region', 'region_id', 'region_a', + 'macrocounty', + 'macrocounty_id', + 'macrocounty_a', 'county', 'county_id', 'county_a', From fd0718a6992f276e429d243e49e2403c5cdebcfc Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 18 Mar 2016 13:12:34 -0400 Subject: [PATCH 091/106] updated labelGeneration special case so that region-specific logic only applies to US/CA --- helper/labelGenerator.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 7e9c1827..967a34c1 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -4,6 +4,8 @@ var _ = require('lodash'), module.exports = function( record ){ var schema = getSchema(record.country_a); + console.log(schema); + var labelParts = getInitialLabel(record); for (var key in schema) { @@ -47,11 +49,28 @@ function getSchema(country_a) { } +// helper function that sets a default label for non-US/CA regions +// this is a very special case function getInitialLabel(record) { - if ('region' === record.layer && ('geonames' === record.source || 'whosonfirst' === record.source)) { + if (isRegion(record.layer) && + isGeonamesOrWhosOnFirst(record.source) && + isUSAOrCAN(record.country_a)) { return []; } return [record.name]; } + +function isRegion(layer) { + return 'region' === layer; +} + +function isUSAOrCAN(country_a) { + return 'USA' === country_a || 'CAN' === country_a; +} + +function isGeonamesOrWhosOnFirst(source) { + return 'geonames' === source || 'whosonfirst' === source; + +} From c746bd8d3637fd0c53576d285ad40e3e4fec38ae Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 18 Mar 2016 13:13:04 -0400 Subject: [PATCH 092/106] added `macroregion` to default and GBR schemas (with tests!) --- helper/labelSchema.js | 9 +++- test/unit/helper/labelGenerator_GBR.js | 17 ++++++++ test/unit/helper/labelGenerator_default.js | 51 ++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/helper/labelSchema.js b/helper/labelSchema.js index 9e5e9030..56161856 100644 --- a/helper/labelSchema.js +++ b/helper/labelSchema.js @@ -8,7 +8,7 @@ module.exports = { 'country': getFirstProperty(['country_a']) }, 'GBR': { - 'local': getFirstProperty(['neighbourhood', 'county', 'localadmin', 'locality', 'region']), + 'local': getFirstProperty(['neighbourhood', 'county', 'localadmin', 'locality', 'macroregion', 'region']), 'regional': getFirstProperty(['county','country','region']) }, 'SGP': { @@ -20,7 +20,7 @@ module.exports = { 'regional': getFirstProperty(['country']) }, 'default': { - 'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county', 'region']), + 'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county', 'macroregion', 'region']), 'regional': getFirstProperty(['country']) } }; @@ -28,10 +28,15 @@ module.exports = { // find the first field of record that has a non-empty value that's not already in labelParts function getFirstProperty(fields) { return function(record, labelParts) { + console.log(record); + console.log(); + for (var i = 0; i < fields.length; i++) { + console.log('considering ' + fields[i]); var fieldValue = record[fields[i]]; if (check.nonEmptyString(fieldValue) && !_.includes(labelParts, fieldValue)) { + console.log('adding ' + fields[i] + ' value: ' + fieldValue); labelParts.push( fieldValue ); return labelParts; } diff --git a/test/unit/helper/labelGenerator_GBR.js b/test/unit/helper/labelGenerator_GBR.js index a121444c..a0f74312 100644 --- a/test/unit/helper/labelGenerator_GBR.js +++ b/test/unit/helper/labelGenerator_GBR.js @@ -58,6 +58,23 @@ module.exports.tests.wales = function(test, common) { }); }; +// GBR macroregion +module.exports.tests.macroregion_trumps_region = function(test, common) { + test('macroregion should trump region when none of neighbourhood, county, localadmin, locality are available', function(t) { + var doc = { + 'name': 'Name', + 'country_a': 'GBR', + 'country': 'Country Name', + 'macroregion': 'Macroregion Name', + 'region': 'Region Name' + }; + + t.equal(generator(doc), 'Name, Macroregion Name, Country Name'); + t.end(); + + }); +}; + module.exports.all = function (tape, common) { function test(name, testFunction) { diff --git a/test/unit/helper/labelGenerator_default.js b/test/unit/helper/labelGenerator_default.js index 789233ba..cbb76f05 100644 --- a/test/unit/helper/labelGenerator_default.js +++ b/test/unit/helper/labelGenerator_default.js @@ -251,6 +251,57 @@ module.exports.tests.waiotapu = function(test, common) { }); }; + +module.exports.tests.non_us_or_ca_region = function(test, common) { + test('geonames US', function(t) { + var doc = { + 'name': 'Default Name', + 'country_a': 'XYZ', + 'country': 'Full Country Name', + 'region': 'Full Region Name', + 'layer': 'region', + 'source': 'geonames' + }; + + t.equal(generator(doc), 'Default Name, Full Region Name, Full Country Name'); + t.end(); + + }); + + test('whosonfirst US', function(t) { + var doc = { + 'name': 'Default Name', + 'country_a': 'XYZ', + 'country': 'Full Country Name', + 'region': 'Full Region Name', + 'layer': 'region', + 'source': 'whosonfirst' + }; + + t.equal(generator(doc), 'Default Name, Full Region Name, Full Country Name'); + t.end(); + + }); + +}; + +// macroregion +module.exports.tests.macroregion_trumps_region = function(test, common) { + test('macroregion should trump region when none of localadmin, locality, neighbourhood, county are available', function(t) { + var doc = { + 'name': 'Name', + 'country_a': 'Country abbreviation', + 'country': 'Country Name', + 'macroregion': 'Macroregion Name', + 'region': 'Region Name' + }; + + t.equal(generator(doc), 'Name, Macroregion Name, Country Name'); + t.end(); + + }); +}; + module.exports.all = function (tape, common) { function test(name, testFunction) { From 5b934b27c30b86e0fa6960f60bd0b9b8b5ad1abb Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 18 Mar 2016 13:37:24 -0400 Subject: [PATCH 093/106] removed debug --- helper/labelGenerator.js | 2 -- helper/labelSchema.js | 5 ----- 2 files changed, 7 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 967a34c1..0f0eb9dd 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -4,8 +4,6 @@ var _ = require('lodash'), module.exports = function( record ){ var schema = getSchema(record.country_a); - console.log(schema); - var labelParts = getInitialLabel(record); for (var key in schema) { diff --git a/helper/labelSchema.js b/helper/labelSchema.js index 56161856..5d3eba5d 100644 --- a/helper/labelSchema.js +++ b/helper/labelSchema.js @@ -28,15 +28,10 @@ module.exports = { // find the first field of record that has a non-empty value that's not already in labelParts function getFirstProperty(fields) { return function(record, labelParts) { - console.log(record); - console.log(); - for (var i = 0; i < fields.length; i++) { - console.log('considering ' + fields[i]); var fieldValue = record[fields[i]]; if (check.nonEmptyString(fieldValue) && !_.includes(labelParts, fieldValue)) { - console.log('adding ' + fields[i] + ' value: ' + fieldValue); labelParts.push( fieldValue ); return labelParts; } From 7d71e3210424182556e54284de51930fab5d43cd Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 24 Mar 2016 14:01:37 +0100 Subject: [PATCH 094/106] bugfix: enable support for ?sources= filtering on /reverse --- query/reverse.js | 1 + test/ciao/reverse/layers_alias_address.coffee | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 test/ciao/reverse/layers_alias_address.coffee diff --git a/query/reverse.js b/query/reverse.js index 856b9540..3e9c3aa5 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -15,6 +15,7 @@ query.sort( peliasQuery.view.sort_distance ); // non-scoring hard filters query.filter( peliasQuery.view.boundary_circle ); +query.filter( peliasQuery.view.sources ); // -------------------------------- diff --git a/test/ciao/reverse/layers_alias_address.coffee b/test/ciao/reverse/layers_alias_address.coffee new file mode 100644 index 00000000..fb8e3262 --- /dev/null +++ b/test/ciao/reverse/layers_alias_address.coffee @@ -0,0 +1,33 @@ + +#> layer alias +path: '/v1/reverse?point.lat=1&point.lon=2&layers=address' + +#? 200 ok +response.statusCode.should.be.equal 200 +response.should.have.header 'charset', 'utf8' +response.should.have.header 'content-type', 'application/json; charset=utf-8' + +#? valid geocoding block +should.exist json.geocoding +should.exist json.geocoding.version +should.exist json.geocoding.attribution +should.exist json.geocoding.query +should.exist json.geocoding.engine +should.exist json.geocoding.engine.name +should.exist json.geocoding.engine.author +should.exist json.geocoding.engine.version +should.exist json.geocoding.timestamp + +#? valid geojson +json.type.should.be.equal 'FeatureCollection' +json.features.should.be.instanceof Array + +#? expected errors +should.not.exist json.geocoding.errors + +#? expected warnings +should.not.exist json.geocoding.warnings + +#? inputs +json.geocoding.query['size'].should.eql 10 +json.geocoding.query.layers.should.eql ["address"] From 63301afeac5a28d6e9cf1b5a79fb00c2befa60e0 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 24 Mar 2016 14:12:52 +0100 Subject: [PATCH 095/106] wow this is easy to get wrong, added a unit test --- query/reverse.js | 3 ++ query/search.js | 1 + .../fixture/reverse_with_source_filtering.js | 51 +++++++++++++++++++ test/unit/query/reverse.js | 17 +++++++ 4 files changed, 72 insertions(+) create mode 100644 test/unit/fixture/reverse_with_source_filtering.js diff --git a/query/reverse.js b/query/reverse.js index 3e9c3aa5..13e9263c 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -28,6 +28,9 @@ function generateQuery( clean ){ vs.var( 'size', clean.querySize); } + // sources + vs.var( 'sources', clean.sources); + // focus point to score by distance if( check.number(clean['point.lat']) && check.number(clean['point.lon']) ){ diff --git a/query/search.js b/query/search.js index 1db1a10a..9f0a792c 100644 --- a/query/search.js +++ b/query/search.js @@ -51,6 +51,7 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); + // sources vs.var( 'sources', clean.sources); // size diff --git a/test/unit/fixture/reverse_with_source_filtering.js b/test/unit/fixture/reverse_with_source_filtering.js new file mode 100644 index 00000000..28fce18b --- /dev/null +++ b/test/unit/fixture/reverse_with_source_filtering.js @@ -0,0 +1,51 @@ +var vs = require('../../../query/reverse_defaults'); + +module.exports = { + 'query': { + 'filtered': { + 'query': { + 'bool': { + 'must': [] + } + }, + 'filter': { + 'bool': { + 'must': [ + { + 'geo_distance': { + 'distance': '500km', + 'distance_type': 'plane', + 'optimize_bbox': 'indexed', + '_cache': true, + 'center_point': { + 'lat': 29.49136, + 'lon': -82.50622 + } + } + }, + { + 'terms': { + 'source': ['test'] + } + } + ] + } + } + } + }, + 'sort': [ + '_score', + { + '_geo_distance': { + 'center_point': { + 'lat': 29.49136, + 'lon': -82.50622 + }, + 'order': 'asc', + 'distance_type': 'plane' + } + } + ], + 'size': vs.size, + 'track_scores': true +}; diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js index 7e6c2ba1..4060d679 100644 --- a/test/unit/query/reverse.js +++ b/test/unit/query/reverse.js @@ -108,6 +108,23 @@ module.exports.tests.query = function(test, common) { t.deepEqual(compiled, expected, 'valid reverse query with boundary.country'); t.end(); }); + + test('valid sources filter', function(t) { + var query = generate({ + 'point.lat': 29.49136, + 'point.lon': -82.50622, + 'boundary.circle.lat': 29.49136, + 'boundary.circle.lon': -82.50622, + 'boundary.circle.radius': 500, + 'sources': ['test'] + }); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/reverse_with_source_filtering'); + + t.deepEqual(compiled, expected, 'valid reverse query with source filtering'); + t.end(); + }); }; module.exports.all = function (tape, common) { From 51279ce2cf75c2760a3947e359cd412b491850d7 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 29 Mar 2016 16:11:43 +0200 Subject: [PATCH 096/106] quattroshapes deprecation warning and wof fallback --- sanitiser/_deprecate_quattroshapes.js | 41 ++++++++++++ sanitiser/_warn_quattroshapes.js | 25 ------- sanitiser/reverse.js | 2 +- sanitiser/search.js | 2 +- .../sources_deprecation_warning.coffee | 34 ++++++++++ .../sources_layers_invalid_combo.coffee | 2 +- .../search/sources_deprecation_warning.coffee | 35 ++++++++++ .../sources_layers_invalid_combo.coffee | 2 +- test/unit/run.js | 1 + .../sanitiser/_deprecate_quattroshapes.js | 65 +++++++++++++++++++ test/unit/sanitiser/reverse.js | 4 +- test/unit/sanitiser/search.js | 4 +- 12 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 sanitiser/_deprecate_quattroshapes.js delete mode 100644 sanitiser/_warn_quattroshapes.js create mode 100644 test/ciao/reverse/sources_deprecation_warning.coffee create mode 100644 test/ciao/search/sources_deprecation_warning.coffee create mode 100644 test/unit/sanitiser/_deprecate_quattroshapes.js diff --git a/sanitiser/_deprecate_quattroshapes.js b/sanitiser/_deprecate_quattroshapes.js new file mode 100644 index 00000000..ad724bbc --- /dev/null +++ b/sanitiser/_deprecate_quattroshapes.js @@ -0,0 +1,41 @@ +var _ = require('lodash'); + +/** + In the process of phasing out the 'quattroshapes' source in favour of 'whosonfirst' + we will emit a warning to users so they can begin upgrading their clients. + + In the interim we will automatically rewrite all requests for quattroshapes to whosonfirst. + + @todo: this is only temporary + @see: https://github.com/pelias/api/issues/442 +**/ + +function sanitize( raw, clean, opts ) { + // error & warning messages + var messages = { errors: [], warnings: [] }; + + // only applicably when 'sources' param is privided + if( raw.hasOwnProperty('sources') ){ + + var sources = raw.sources.split(','); + if (_.includes(sources, 'quattroshapes') || _.includes(sources, 'qs')) { + + // emit a warning message so users can transition. + messages.warnings.push('You are using Quattroshapes as a data source in this query. ' + + 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + + 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + + 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + + 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + + 'be deprecated. If you have any questions, please email search@mapzen.com.'); + + // user requested 'quattroshapes', we will give them 'whosonfirst' instead. + sources = _.without(sources, 'quattroshapes', 'qs'); + sources.push('whosonfirst'); + raw.sources = sources.join(','); + } + } + + return messages; +} + +module.exports = sanitize; diff --git a/sanitiser/_warn_quattroshapes.js b/sanitiser/_warn_quattroshapes.js deleted file mode 100644 index 2b887e5f..00000000 --- a/sanitiser/_warn_quattroshapes.js +++ /dev/null @@ -1,25 +0,0 @@ -var _ = require('lodash'); - -function setup( paramName, targetMap ) { - return function( raw, clean ){ - return sanitize( raw, clean ); - }; -} - -function sanitize( raw, clean, opts ) { - // error & warning messages - var messages = { errors: [], warnings: [] }; - - if (_.includes(raw.sources, 'quattroshapes') || _.includes(raw.sources, 'qs')) { - messages.warnings.push( 'You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + - 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + - 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + - 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + - 'be deprecated. If you have any questions, please email search@mapzen.com.'); - } - - return messages; -} - -module.exports = setup; diff --git a/sanitiser/reverse.js b/sanitiser/reverse.js index 5f0ffb6b..c8e1e3f4 100644 --- a/sanitiser/reverse.js +++ b/sanitiser/reverse.js @@ -2,12 +2,12 @@ var type_mapping = require('../helper/type_mapping'); var sanitizeAll = require('../sanitiser/sanitizeAll'), sanitizers = { + quattroshapes_deprecation: require('../sanitiser/_deprecate_quattroshapes'), singleScalarParameters: require('../sanitiser/_single_scalar_parameters'), layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitisers, must be run after them sources_and_layers: require('../sanitiser/_sources_and_layers'), - quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(), size: require('../sanitiser/_size'), private: require('../sanitiser/_flag_bool')('private', false), geo_reverse: require('../sanitiser/_geo_reverse'), diff --git a/sanitiser/search.js b/sanitiser/search.js index 1388505a..8b26e54e 100644 --- a/sanitiser/search.js +++ b/sanitiser/search.js @@ -2,6 +2,7 @@ var type_mapping = require('../helper/type_mapping'); var sanitizeAll = require('../sanitiser/sanitizeAll'), sanitizers = { + quattroshapes_deprecation: require('../sanitiser/_deprecate_quattroshapes'), singleScalarParameters: require('../sanitiser/_single_scalar_parameters'), text: require('../sanitiser/_text'), size: require('../sanitiser/_size'), @@ -9,7 +10,6 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'), sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitisers, must be run after them sources_and_layers: require('../sanitiser/_sources_and_layers'), - quattroshapes_warning: require('../sanitiser/_warn_quattroshapes')(), private: require('../sanitiser/_flag_bool')('private', false), geo_search: require('../sanitiser/_geo_search'), boundary_country: require('../sanitiser/_boundary_country'), diff --git a/test/ciao/reverse/sources_deprecation_warning.coffee b/test/ciao/reverse/sources_deprecation_warning.coffee new file mode 100644 index 00000000..c6e8d488 --- /dev/null +++ b/test/ciao/reverse/sources_deprecation_warning.coffee @@ -0,0 +1,34 @@ + +#> quattroshapes is being phased out and so should emit a warning message +path: '/v1/reverse?point.lat=1&point.lon=2&sources=qs' + +#? 200 ok +response.statusCode.should.be.equal 200 +response.should.have.header 'charset', 'utf8' +response.should.have.header 'content-type', 'application/json; charset=utf-8' + +#? valid geocoding block +should.exist json.geocoding +should.exist json.geocoding.version +should.exist json.geocoding.attribution +should.exist json.geocoding.query +should.exist json.geocoding.engine +should.exist json.geocoding.engine.name +should.exist json.geocoding.engine.author +should.exist json.geocoding.engine.version +should.exist json.geocoding.timestamp + +#? valid geojson +json.type.should.be.equal 'FeatureCollection' +json.features.should.be.instanceof Array + +#? expected errors +should.not.exist json.geocoding.errors + +#? expected warnings +should.exist json.geocoding.warnings +json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] + +#? inputs +json.geocoding.query['size'].should.eql 10 +json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/ciao/reverse/sources_layers_invalid_combo.coffee b/test/ciao/reverse/sources_layers_invalid_combo.coffee index 565a68b1..baf157cc 100644 --- a/test/ciao/reverse/sources_layers_invalid_combo.coffee +++ b/test/ciao/reverse/sources_layers_invalid_combo.coffee @@ -27,7 +27,7 @@ should.exist json.geocoding.errors json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ] #? expected warnings -json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] +should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 diff --git a/test/ciao/search/sources_deprecation_warning.coffee b/test/ciao/search/sources_deprecation_warning.coffee new file mode 100644 index 00000000..bfaad028 --- /dev/null +++ b/test/ciao/search/sources_deprecation_warning.coffee @@ -0,0 +1,35 @@ + +#> quattroshapes is being phased out and so should emit a warning message +path: '/v1/search?sources=qs&text=a' + +#? 200 ok +response.statusCode.should.be.equal 200 +response.should.have.header 'charset', 'utf8' +response.should.have.header 'content-type', 'application/json; charset=utf-8' + +#? valid geocoding block +should.exist json.geocoding +should.exist json.geocoding.version +should.exist json.geocoding.attribution +should.exist json.geocoding.query +should.exist json.geocoding.engine +should.exist json.geocoding.engine.name +should.exist json.geocoding.engine.author +should.exist json.geocoding.engine.version +should.exist json.geocoding.timestamp + +#? valid geojson +json.type.should.be.equal 'FeatureCollection' +json.features.should.be.instanceof Array + +#? expected errors +should.not.exist json.geocoding.errors + +#? expected warnings +should.exist json.geocoding.warnings +json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] + +#? inputs +json.geocoding.query['size'].should.eql 10 +json.geocoding.query['text'].should.eql 'a' +json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/ciao/search/sources_layers_invalid_combo.coffee b/test/ciao/search/sources_layers_invalid_combo.coffee index b2880822..821bf243 100644 --- a/test/ciao/search/sources_layers_invalid_combo.coffee +++ b/test/ciao/search/sources_layers_invalid_combo.coffee @@ -27,7 +27,7 @@ should.exist json.geocoding.errors json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ] #? expected warnings -json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] +should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['text'].should.eql 'a' diff --git a/test/unit/run.js b/test/unit/run.js index d7dfd23a..2e8b382e 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -46,6 +46,7 @@ var tests = [ require('./sanitiser/_sources'), require('./sanitiser/_sources_and_layers'), require('./sanitiser/_text'), + require('./sanitiser/_deprecate_quattroshapes'), require('./src/backend'), require('./sanitiser/autocomplete'), require('./sanitiser/place'), diff --git a/test/unit/sanitiser/_deprecate_quattroshapes.js b/test/unit/sanitiser/_deprecate_quattroshapes.js new file mode 100644 index 00000000..8c8a6b95 --- /dev/null +++ b/test/unit/sanitiser/_deprecate_quattroshapes.js @@ -0,0 +1,65 @@ +var sanitize = require('../../../sanitiser/_deprecate_quattroshapes'); + +module.exports.tests = {}; + +module.exports.tests.warning_message_1 = function(test, common) { + test('[qs] should emit a deprecation warning', function(t) { + var raw = { sources: 'qs' }; + var clean = {}; + + var messages = sanitize(raw, clean); + t.deepEquals(messages, { + errors: [], + warnings: ['You are using Quattroshapes as a data source in this query. ' + + 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + + 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + + 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + + 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + + 'be deprecated. If you have any questions, please email search@mapzen.com.'] + }, 'warning emitted'); + + t.end(); + }); +}; + +module.exports.tests.warning_message_2 = function(test, common) { + test('[quattroshapes] should emit a deprecation warning', function(t) { + var raw = { sources: 'quattroshapes' }; + var clean = {}; + + var messages = sanitize(raw, clean); + t.deepEquals(messages, { + errors: [], + warnings: ['You are using Quattroshapes as a data source in this query. ' + + 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + + 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + + 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + + 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + + 'be deprecated. If you have any questions, please email search@mapzen.com.'] + }, 'warning emitted'); + + t.end(); + }); +}; + +module.exports.tests.rewrite = function(test, common) { + test('should rewrite qs and quattroshapes to whosonfirst', function(t) { + var raw = { sources: 'qs,quattroshapes,qs,quattroshapes,osm' }; + var clean = {}; + + sanitize(raw, clean); + t.equals(raw.sources,'osm,whosonfirst','use wof instead of qs'); + + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANTIZE _deprecate_quattroshapes ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/sanitiser/reverse.js b/test/unit/sanitiser/reverse.js index f082cb11..fcce8226 100644 --- a/test/unit/sanitiser/reverse.js +++ b/test/unit/sanitiser/reverse.js @@ -36,8 +36,8 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.sanitisers = function(test, common) { test('check sanitiser list', function (t) { - var expected = ['singleScalarParameters', 'layers', 'sources', 'sources_and_layers', - 'quattroshapes_warning', 'size', 'private', 'geo_reverse', 'boundary_country']; + var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers', + 'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country']; t.deepEqual(Object.keys(reverse.sanitiser_list), expected); t.end(); }); diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index 03a41434..e09672ce 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -24,8 +24,8 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.sanitisers = function(test, common) { test('check sanitiser list', function (t) { - var expected = ['singleScalarParameters', 'text', 'size', 'layers', 'sources', - 'sources_and_layers', 'quattroshapes_warning', 'private', 'geo_search', 'boundary_country' ]; + var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'text', 'size', + 'layers', 'sources', 'sources_and_layers', 'private', 'geo_search', 'boundary_country' ]; t.deepEqual(Object.keys(search.sanitiser_list), expected); t.end(); }); From abbd314ee833f641c4dda7222c18b968757c2f37 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 30 Mar 2016 16:03:14 -0400 Subject: [PATCH 097/106] Change deprication message to present tense --- sanitiser/_deprecate_quattroshapes.js | 10 +++++----- .../sources_deprecation_warning.coffee | 2 +- .../search/sources_deprecation_warning.coffee | 2 +- .../sanitiser/_deprecate_quattroshapes.js | 20 +++++++++---------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sanitiser/_deprecate_quattroshapes.js b/sanitiser/_deprecate_quattroshapes.js index ad724bbc..bd45c025 100644 --- a/sanitiser/_deprecate_quattroshapes.js +++ b/sanitiser/_deprecate_quattroshapes.js @@ -22,11 +22,11 @@ function sanitize( raw, clean, opts ) { // emit a warning message so users can transition. messages.warnings.push('You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + - 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + - 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + - 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + - 'be deprecated. If you have any questions, please email search@mapzen.com.'); + 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + + 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + + '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'); // user requested 'quattroshapes', we will give them 'whosonfirst' instead. sources = _.without(sources, 'quattroshapes', 'qs'); diff --git a/test/ciao/reverse/sources_deprecation_warning.coffee b/test/ciao/reverse/sources_deprecation_warning.coffee index c6e8d488..45c2c967 100644 --- a/test/ciao/reverse/sources_deprecation_warning.coffee +++ b/test/ciao/reverse/sources_deprecation_warning.coffee @@ -27,7 +27,7 @@ should.not.exist json.geocoding.errors #? expected warnings should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] +json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ] #? inputs json.geocoding.query['size'].should.eql 10 diff --git a/test/ciao/search/sources_deprecation_warning.coffee b/test/ciao/search/sources_deprecation_warning.coffee index bfaad028..c6d96a2c 100644 --- a/test/ciao/search/sources_deprecation_warning.coffee +++ b/test/ciao/search/sources_deprecation_warning.coffee @@ -27,7 +27,7 @@ should.not.exist json.geocoding.errors #? expected warnings should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql [ 'You are using Quattroshapes as a data source in this query. Quattroshapes will be disabled as a data source for Mapzen Search in the next several weeks, and is being replaced by Who\'s on First, an actively maintained data project based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be deprecated. If you have any questions, please email search@mapzen.com.' ] +json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ] #? inputs json.geocoding.query['size'].should.eql 10 diff --git a/test/unit/sanitiser/_deprecate_quattroshapes.js b/test/unit/sanitiser/_deprecate_quattroshapes.js index 8c8a6b95..9a4ab53d 100644 --- a/test/unit/sanitiser/_deprecate_quattroshapes.js +++ b/test/unit/sanitiser/_deprecate_quattroshapes.js @@ -11,11 +11,11 @@ module.exports.tests.warning_message_1 = function(test, common) { t.deepEquals(messages, { errors: [], warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + - 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + - 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + - 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + - 'be deprecated. If you have any questions, please email search@mapzen.com.'] + 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + + 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + + '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'] }, 'warning emitted'); t.end(); @@ -31,11 +31,11 @@ module.exports.tests.warning_message_2 = function(test, common) { t.deepEquals(messages, { errors: [], warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes will be disabled as a data source for Mapzen Search in the next several ' + - 'weeks, and is being replaced by Who\'s on First, an actively maintained data project ' + - 'based on Quattroshapes. Your existing queries WILL CONTINUE TO WORK for the foreseeable ' + - 'future, but results will be coming from Who\'s on First and `sources=quattroshapes` will ' + - 'be deprecated. If you have any questions, please email search@mapzen.com.'] + 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + + 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + + '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'] }, 'warning emitted'); t.end(); From 7ebb653c22fbcad6d7b73e1cc024445621041cf5 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Mon, 4 Apr 2016 22:20:49 -0400 Subject: [PATCH 098/106] Change properties.bounding_box to bbox at top level --- helper/geojsonify.js | 34 ++++++++++++++++++++++++++++++++-- test/unit/helper/geojsonify.js | 7 +------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 1f9a60a8..a6e082eb 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -35,8 +35,7 @@ var DETAILS_PROPS = [ 'locality_id', 'locality_a', 'neighbourhood', - 'neighbourhood_id', - 'bounding_box' + 'neighbourhood_id' ]; @@ -65,6 +64,10 @@ function geojsonifyPlaces( docs ){ var geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] }); var geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] }); + // to insert the bbox property at the top level of each feature, it must be done separately after + // initial geojson construction is finished + addBBoxPerFeature(geojson); + // bounding box calculations computeBBox(geojson, geojsonExtentPoints); @@ -117,6 +120,30 @@ function addLabel(src, dst) { dst.label = labelGenerator(dst); } +/** + * Add bounding box + * + * @param {object} geojson + */ +function addBBoxPerFeature(geojson) { + geojson.features.forEach(function (feature) { + + if (!feature.properties.hasOwnProperty('bounding_box')) { + return; + } + + if (feature.properties.bounding_box) { + feature.bbox = [ + feature.properties.bounding_box.min_lon, + feature.properties.bounding_box.min_lat, + feature.properties.bounding_box.max_lon, + feature.properties.bounding_box.max_lat + ]; + } + + delete feature.properties.bounding_box; + }); +} /** * Collect all points from the geodata. @@ -222,6 +249,9 @@ function addMetaData(src, dst) { dst.gid = makeGid(src); dst.layer = lookupLayer(src); dst.source = lookupSource(src); + if (src.hasOwnProperty('bounding_box')) { + dst.bounding_box = src.bounding_box; + } } /** diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 7b371b27..132f1e97 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -333,14 +333,9 @@ module.exports.tests.search = function(test, common) { 'localadmin_id': '404521211', 'locality': 'New York', 'locality_id': '85977539', - 'bounding_box': { - 'min_lat': 40.6514712164, - 'max_lat': 40.6737320588, - 'min_lon': -73.8967895508, - 'max_lon': -73.8665771484 - }, 'label': 'East New York, Brooklyn, NY, USA' }, + 'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588], 'geometry': { 'type': 'Point', 'coordinates': [ From 08684947784c0c9908efb2aec9ecfb494834ec07 Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Wed, 6 Apr 2016 10:42:22 -0700 Subject: [PATCH 099/106] chore(package): update tap-dot to version 1.0.5 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 49e00498..84ada6ea 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "nsp": "^2.2.0", "precommit-hook": "^3.0.0", "proxyquire": "^1.4.0", - "tap-dot": "1.0.4", + "tap-dot": "1.0.5", "tape": "^4.4.0" }, "pre-commit": [ From 7231e229d4ed21c0088e1a0e6054cfe806497a35 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Wed, 6 Apr 2016 19:27:38 -0400 Subject: [PATCH 100/106] Convert parent WOF ids to valid Pelias ids to be used by /place --- middleware/normalizeParentIds.js | 66 +++++++++++++++++ package.json | 1 + routes/v1.js | 7 +- test/unit/middleware/normalizeParentIds.js | 85 ++++++++++++++++++++++ test/unit/run.js | 1 + 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 middleware/normalizeParentIds.js create mode 100644 test/unit/middleware/normalizeParentIds.js diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js new file mode 100644 index 00000000..2c9d3629 --- /dev/null +++ b/middleware/normalizeParentIds.js @@ -0,0 +1,66 @@ +var logger = require('pelias-logger').get('api'); +var Document = require('pelias-model').Document; + +var placeTypes = [ + 'country', + 'macroregion', + 'region', + 'macrocounty', + 'county', + 'localadmin', + 'locality', + 'neighbourhood', + 'borough' +]; + +/** + * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint. + * This should probably be moved to the import pipeline once we are happy with the way this works. + */ + +function setup() { + return function (req, res, next) { + // do nothing if no result data set + if (!res || !res.data) { + return next(); + } + + res.data = res.data.map(normalizeParentIds); + + next(); + }; +} + +/** + * Update all parent ids in the admin hierarchy + * + * @param {object} place + * @return {object} + */ +function normalizeParentIds(place) { + + if (place && place.parent) { + placeTypes.forEach(function (placeType) { + if (place.parent[placeType] && place.parent[placeType].length > 0 && place.parent[placeType][0]) { + place.parent[placeType + '_id'] = [ makeNewId(placeType, place.parent[placeType + '_id']) ]; + } + }); + } + + return place; +} + +/** + * Generate a valid Pelias ids from placetype and WOF id. + * Assumes all of the incoming ids are WOF ids. + * + * @param {string} placeType + * @param {number} id + * @return {string} + */ +function makeNewId(placeType, id) { + var doc = new Document('whosonfirst', placeType, id); + return doc.getGid(); +} + +module.exports = setup; diff --git a/package.json b/package.json index 84ada6ea..f97df8eb 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", + "pelias-model": "^3.1.0", "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", diff --git a/routes/v1.js b/routes/v1.js index eacd0a40..c8c88de1 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -35,7 +35,8 @@ var postProc = { renamePlacenames: require('../middleware/renamePlacenames'), geocodeJSON: require('../middleware/geocodeJSON'), sendJSON: require('../middleware/sendJSON'), - parseBoundingBox: require('../middleware/parseBBox') + parseBoundingBox: require('../middleware/parseBBox'), + normalizeParentIds: require('../middleware/normalizeParentIds') }; /** @@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -79,6 +81,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -94,6 +97,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -103,6 +107,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js new file mode 100644 index 00000000..9ad603f8 --- /dev/null +++ b/test/unit/middleware/normalizeParentIds.js @@ -0,0 +1,85 @@ +var normalizer = require('../../../middleware/normalizeParentIds')(); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('WOF ids converted to Pelias ids', function(t) { + + var input = { + data: [{ + 'parent': { + 'country': ['United States'], + 'country_id': ['85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] + } + }] + }; + + var expected = { + data: [{ + 'parent': { + 'country': ['United States'], + 'country_id': ['whosonfirst:country:85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['whosonfirst:region:85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['whosonfirst:county:102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] + } + }] + }; + + normalizer({}, input, function () { + t.deepEqual(input, expected); + t.end(); + }); + + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('[middleware] normalizeParentIds: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/run.js b/test/unit/run.js index 2e8b382e..c1ab11b7 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -28,6 +28,7 @@ var tests = [ require('./middleware/localNamingConventions'), require('./middleware/dedupe'), require('./middleware/parseBBox'), + require('./middleware/normalizeParentIds'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'), From 10cd693e0213f96d1b7998aa10e82961351315ed Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 7 Apr 2016 14:49:22 -0400 Subject: [PATCH 101/106] parent block properties should not be changed, only top level ones --- middleware/normalizeParentIds.js | 6 +- test/unit/middleware/normalizeParentIds.js | 98 ++++++++++++---------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index 2c9d3629..fe512b61 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -39,10 +39,10 @@ function setup() { */ function normalizeParentIds(place) { - if (place && place.parent) { + if (place) { placeTypes.forEach(function (placeType) { - if (place.parent[placeType] && place.parent[placeType].length > 0 && place.parent[placeType][0]) { - place.parent[placeType + '_id'] = [ makeNewId(placeType, place.parent[placeType + '_id']) ]; + if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) { + place[placeType + '_id'] = [ makeNewId(placeType, place[placeType + '_id']) ]; } }); } diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js index 9ad603f8..95dc1387 100644 --- a/test/unit/middleware/normalizeParentIds.js +++ b/test/unit/middleware/normalizeParentIds.js @@ -8,30 +8,33 @@ module.exports.tests.interface = function(test, common) { var input = { data: [{ 'parent': { - 'country': ['United States'], + 'country': ['United States'], // these shouldn't change 'country_id': ['85633793'], - 'country_a': ['USA'], - 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['foobar'], - 'macroregion_a': ['MacroRegion Abbreviation'], - 'region': ['New York'], - 'region_id': ['85688543'], - 'region_a': ['NY'], - 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['~~~~~'], - 'macrocounty_a': ['MacroCounty Abbreviation'], - 'county': ['Kings County'], - 'county_id': ['102082361'], - 'county_a': [null], - 'localadmin': ['Brooklyn'], - 'localadmin_id': ['404521211'], - 'localadmin_a': [null], - 'locality': ['Some Locality'], - 'locality_id': ['85977539'], - 'locality_a': [null], - 'neighbourhood': [], - 'neighbourhood_id': [] - } + 'country_a': ['USA'] + }, + 'country': ['United States'], + 'country_id': ['85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] }] }; @@ -39,29 +42,32 @@ module.exports.tests.interface = function(test, common) { data: [{ 'parent': { 'country': ['United States'], - 'country_id': ['whosonfirst:country:85633793'], - 'country_a': ['USA'], - 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['whosonfirst:macroregion:foobar'], - 'macroregion_a': ['MacroRegion Abbreviation'], - 'region': ['New York'], - 'region_id': ['whosonfirst:region:85688543'], - 'region_a': ['NY'], - 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], - 'macrocounty_a': ['MacroCounty Abbreviation'], - 'county': ['Kings County'], - 'county_id': ['whosonfirst:county:102082361'], - 'county_a': [null], - 'localadmin': ['Brooklyn'], - 'localadmin_id': ['whosonfirst:localadmin:404521211'], - 'localadmin_a': [null], - 'locality': ['Some Locality'], - 'locality_id': ['whosonfirst:locality:85977539'], - 'locality_a': [null], - 'neighbourhood': [], - 'neighbourhood_id': [] - } + 'country_id': ['85633793'], + 'country_a': ['USA'] + }, + 'country': ['United States'], + 'country_id': ['whosonfirst:country:85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['whosonfirst:region:85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['whosonfirst:county:102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] }] }; From 3862dd233edd31ca96a856843c7f909c185da871 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 7 Apr 2016 16:41:08 -0400 Subject: [PATCH 102/106] change admin property names to _gid from _id --- helper/geojsonify.js | 20 ++++++----- helper/placeTypes.js | 11 ++++++ middleware/normalizeParentIds.js | 14 ++------ middleware/renamePlacenames.js | 40 ++-------------------- test/unit/helper/geojsonify.js | 32 +++++++++-------- test/unit/middleware/normalizeParentIds.js | 32 ++++++++--------- 6 files changed, 60 insertions(+), 89 deletions(-) create mode 100644 helper/placeTypes.js diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 1f9a60a8..8018b209 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -4,6 +4,7 @@ var GeoJSON = require('geojson'), labelGenerator = require('./labelGenerator'), logger = require('pelias-logger').get('api'), type_mapping = require('./type_mapping'), + Document = require('pelias-model').Document, _ = require('lodash'); // Properties to be copied @@ -14,28 +15,28 @@ var DETAILS_PROPS = [ 'confidence', 'distance', 'country', - 'country_id', + 'country_gid', 'country_a', 'macroregion', - 'macroregion_id', + 'macroregion_gid', 'macroregion_a', 'region', - 'region_id', + 'region_gid', 'region_a', 'macrocounty', - 'macrocounty_id', + 'macrocounty_gid', 'macrocounty_a', 'county', - 'county_id', + 'county_gid', 'county_a', 'localadmin', - 'localadmin_id', + 'localadmin_gid', 'localadmin_a', 'locality', - 'locality_id', + 'locality_gid', 'locality_a', 'neighbourhood', - 'neighbourhood_id', + 'neighbourhood_gid', 'bounding_box' ]; @@ -208,7 +209,8 @@ function copyProperties( source, props, dst ) { * @param {object} src */ function makeGid(src) { - return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id; + var doc = new Document(lookupSource(src), lookupLayer(src), src._id); + return doc.getGid(); } /** diff --git a/helper/placeTypes.js b/helper/placeTypes.js new file mode 100644 index 00000000..76538982 --- /dev/null +++ b/helper/placeTypes.js @@ -0,0 +1,11 @@ +module.exports = [ + 'country', + 'macroregion', + 'region', + 'macrocounty', + 'county', + 'localadmin', + 'locality', + 'borough', + 'neighbourhood' +]; diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index fe512b61..e0e499e2 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -1,17 +1,7 @@ var logger = require('pelias-logger').get('api'); var Document = require('pelias-model').Document; -var placeTypes = [ - 'country', - 'macroregion', - 'region', - 'macrocounty', - 'county', - 'localadmin', - 'locality', - 'neighbourhood', - 'borough' -]; +var placeTypes = require('../helper/placeTypes'); /** * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint. @@ -42,7 +32,7 @@ function normalizeParentIds(place) { if (place) { placeTypes.forEach(function (placeType) { if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) { - place[placeType + '_id'] = [ makeNewId(placeType, place[placeType + '_id']) ]; + place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ]; } }); } diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 152ad404..d89a43ad 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -1,16 +1,6 @@ var _ = require('lodash'); -/** - - P is a preferred English name - - Q is a preferred name (in other languages) - - V is a well-known (but unofficial) variant for the place - (e.g. "New York City" for New York) - - S is either a synonym or a colloquial name for the place - (e.g. "Big Apple" for New York), or a version of the name which - is stripped of accent characters. - - A is an abbreviation or code for the place (e.g. "NYC" for New - York) - */ +var PARENT_PROPS = require('../helper/placeTypes'); var ADDRESS_PROPS = { 'number': 'housenumber', @@ -18,32 +8,6 @@ var ADDRESS_PROPS = { 'street': 'street' }; -var PARENT_PROPS = [ - 'country', - 'country_id', - 'country_a', - 'macroregion', - 'macroregion_id', - 'macroregion_a', - 'region', - 'region_id', - 'region_a', - 'macrocounty', - 'macrocounty_id', - 'macrocounty_a', - 'county', - 'county_id', - 'county_a', - 'localadmin', - 'localadmin_id', - 'localadmin_a', - 'locality', - 'locality_id', - 'locality_a', - 'neighbourhood', - 'neighbourhood_id' -]; - function setup() { return renamePlacenames; @@ -74,6 +38,8 @@ function renameOneRecord(place) { if (place.parent) { PARENT_PROPS.forEach(function (prop) { place[prop] = place.parent[prop]; + place[prop + '_a'] = place.parent[prop + '_a']; + place[prop + '_gid'] = place.parent[prop + '_id']; }); } diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 7b371b27..3a15d8f5 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) { var earth = [{ '_type': 'geoname', '_id': '6295630', + 'source': 'whosonfirst', + 'layer': 'continent', 'name': { 'default': 'Earth' }, @@ -240,7 +242,7 @@ module.exports.tests.search = function(test, common) { 'country': [ 'United States' ], - 'country_id': [ + 'country_gid': [ '85633793' ], 'country_a': [ @@ -249,7 +251,7 @@ module.exports.tests.search = function(test, common) { 'macroregion': [ 'MacroRegion Name' ], - 'macroregion_id': [ + 'macroregion_gid': [ 'MacroRegion Id' ], 'macroregion_a': [ @@ -258,7 +260,7 @@ module.exports.tests.search = function(test, common) { 'region': [ 'New York' ], - 'region_id': [ + 'region_gid': [ '85688543' ], 'region_a': [ @@ -267,7 +269,7 @@ module.exports.tests.search = function(test, common) { 'macrocounty': [ 'MacroCounty Name' ], - 'macrocounty_id': [ + 'macrocounty_gid': [ 'MacroCounty Id' ], 'macrocounty_a': [ @@ -276,7 +278,7 @@ module.exports.tests.search = function(test, common) { 'county': [ 'Kings County' ], - 'county_id': [ + 'county_gid': [ '102082361' ], 'county_a': [ @@ -285,20 +287,20 @@ module.exports.tests.search = function(test, common) { 'localadmin': [ 'Brooklyn' ], - 'localadmin_id': [ + 'localadmin_gid': [ '404521211' ], 'localadmin_a': [ null ], - 'locality_id': [ + 'locality_gid': [ '85977539' ], 'locality_a': [ null ], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] } ]; @@ -316,23 +318,23 @@ module.exports.tests.search = function(test, common) { 'name': 'East New York', 'confidence': 0.888, 'country': 'United States', - 'country_id': '85633793', + 'country_gid': '85633793', 'country_a': 'USA', 'macroregion': 'MacroRegion Name', - 'macroregion_id': 'MacroRegion Id', + 'macroregion_gid': 'MacroRegion Id', 'macroregion_a': 'MacroRegion Abbreviation', 'region': 'New York', - 'region_id': '85688543', + 'region_gid': '85688543', 'region_a': 'NY', 'macrocounty': 'MacroCounty Name', - 'macrocounty_id': 'MacroCounty Id', + 'macrocounty_gid': 'MacroCounty Id', 'macrocounty_a': 'MacroCounty Abbreviation', 'county': 'Kings County', - 'county_id': '102082361', + 'county_gid': '102082361', 'localadmin': 'Brooklyn', - 'localadmin_id': '404521211', + 'localadmin_gid': '404521211', 'locality': 'New York', - 'locality_id': '85977539', + 'locality_gid': '85977539', 'bounding_box': { 'min_lat': 40.6514712164, 'max_lat': 40.6737320588, diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js index 95dc1387..472df0e3 100644 --- a/test/unit/middleware/normalizeParentIds.js +++ b/test/unit/middleware/normalizeParentIds.js @@ -13,28 +13,28 @@ module.exports.tests.interface = function(test, common) { 'country_a': ['USA'] }, 'country': ['United States'], - 'country_id': ['85633793'], + 'country_gid': ['85633793'], 'country_a': ['USA'], 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['foobar'], + 'macroregion_gid': ['foobar'], 'macroregion_a': ['MacroRegion Abbreviation'], 'region': ['New York'], - 'region_id': ['85688543'], + 'region_gid': ['85688543'], 'region_a': ['NY'], 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['~~~~~'], + 'macrocounty_gid': ['~~~~~'], 'macrocounty_a': ['MacroCounty Abbreviation'], 'county': ['Kings County'], - 'county_id': ['102082361'], + 'county_gid': ['102082361'], 'county_a': [null], 'localadmin': ['Brooklyn'], - 'localadmin_id': ['404521211'], + 'localadmin_gid': ['404521211'], 'localadmin_a': [null], 'locality': ['Some Locality'], - 'locality_id': ['85977539'], + 'locality_gid': ['85977539'], 'locality_a': [null], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] }] }; @@ -46,28 +46,28 @@ module.exports.tests.interface = function(test, common) { 'country_a': ['USA'] }, 'country': ['United States'], - 'country_id': ['whosonfirst:country:85633793'], + 'country_gid': ['whosonfirst:country:85633793'], 'country_a': ['USA'], 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_gid': ['whosonfirst:macroregion:foobar'], 'macroregion_a': ['MacroRegion Abbreviation'], 'region': ['New York'], - 'region_id': ['whosonfirst:region:85688543'], + 'region_gid': ['whosonfirst:region:85688543'], 'region_a': ['NY'], 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_gid': ['whosonfirst:macrocounty:~~~~~'], 'macrocounty_a': ['MacroCounty Abbreviation'], 'county': ['Kings County'], - 'county_id': ['whosonfirst:county:102082361'], + 'county_gid': ['whosonfirst:county:102082361'], 'county_a': [null], 'localadmin': ['Brooklyn'], - 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_gid': ['whosonfirst:localadmin:404521211'], 'localadmin_a': [null], 'locality': ['Some Locality'], - 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_gid': ['whosonfirst:locality:85977539'], 'locality_a': [null], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] }] }; From 201f4d4fb32f0f942437d5f197d72053a6fedbab Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Wed, 6 Apr 2016 19:27:38 -0400 Subject: [PATCH 103/106] Convert parent WOF ids to valid Pelias ids to be used by /place --- middleware/normalizeParentIds.js | 66 +++++++++++++++++ package.json | 1 + routes/v1.js | 7 +- test/unit/middleware/normalizeParentIds.js | 85 ++++++++++++++++++++++ test/unit/run.js | 1 + 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 middleware/normalizeParentIds.js create mode 100644 test/unit/middleware/normalizeParentIds.js diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js new file mode 100644 index 00000000..2c9d3629 --- /dev/null +++ b/middleware/normalizeParentIds.js @@ -0,0 +1,66 @@ +var logger = require('pelias-logger').get('api'); +var Document = require('pelias-model').Document; + +var placeTypes = [ + 'country', + 'macroregion', + 'region', + 'macrocounty', + 'county', + 'localadmin', + 'locality', + 'neighbourhood', + 'borough' +]; + +/** + * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint. + * This should probably be moved to the import pipeline once we are happy with the way this works. + */ + +function setup() { + return function (req, res, next) { + // do nothing if no result data set + if (!res || !res.data) { + return next(); + } + + res.data = res.data.map(normalizeParentIds); + + next(); + }; +} + +/** + * Update all parent ids in the admin hierarchy + * + * @param {object} place + * @return {object} + */ +function normalizeParentIds(place) { + + if (place && place.parent) { + placeTypes.forEach(function (placeType) { + if (place.parent[placeType] && place.parent[placeType].length > 0 && place.parent[placeType][0]) { + place.parent[placeType + '_id'] = [ makeNewId(placeType, place.parent[placeType + '_id']) ]; + } + }); + } + + return place; +} + +/** + * Generate a valid Pelias ids from placetype and WOF id. + * Assumes all of the incoming ids are WOF ids. + * + * @param {string} placeType + * @param {number} id + * @return {string} + */ +function makeNewId(placeType, id) { + var doc = new Document('whosonfirst', placeType, id); + return doc.getGid(); +} + +module.exports = setup; diff --git a/package.json b/package.json index 84ada6ea..f97df8eb 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", + "pelias-model": "^3.1.0", "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", diff --git a/routes/v1.js b/routes/v1.js index eacd0a40..c8c88de1 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -35,7 +35,8 @@ var postProc = { renamePlacenames: require('../middleware/renamePlacenames'), geocodeJSON: require('../middleware/geocodeJSON'), sendJSON: require('../middleware/sendJSON'), - parseBoundingBox: require('../middleware/parseBBox') + parseBoundingBox: require('../middleware/parseBBox'), + normalizeParentIds: require('../middleware/normalizeParentIds') }; /** @@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -79,6 +81,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -94,6 +97,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), @@ -103,6 +107,7 @@ function addRoutes(app, peliasConfig) { postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), + postProc.normalizeParentIds(), postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js new file mode 100644 index 00000000..9ad603f8 --- /dev/null +++ b/test/unit/middleware/normalizeParentIds.js @@ -0,0 +1,85 @@ +var normalizer = require('../../../middleware/normalizeParentIds')(); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('WOF ids converted to Pelias ids', function(t) { + + var input = { + data: [{ + 'parent': { + 'country': ['United States'], + 'country_id': ['85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] + } + }] + }; + + var expected = { + data: [{ + 'parent': { + 'country': ['United States'], + 'country_id': ['whosonfirst:country:85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['whosonfirst:region:85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['whosonfirst:county:102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] + } + }] + }; + + normalizer({}, input, function () { + t.deepEqual(input, expected); + t.end(); + }); + + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('[middleware] normalizeParentIds: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/run.js b/test/unit/run.js index 2e8b382e..c1ab11b7 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -28,6 +28,7 @@ var tests = [ require('./middleware/localNamingConventions'), require('./middleware/dedupe'), require('./middleware/parseBBox'), + require('./middleware/normalizeParentIds'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'), From 82e39fd81f103f21877bbb539750264813e402e3 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 7 Apr 2016 14:49:22 -0400 Subject: [PATCH 104/106] parent block properties should not be changed, only top level ones --- middleware/normalizeParentIds.js | 6 +- test/unit/middleware/normalizeParentIds.js | 98 ++++++++++++---------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index 2c9d3629..fe512b61 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -39,10 +39,10 @@ function setup() { */ function normalizeParentIds(place) { - if (place && place.parent) { + if (place) { placeTypes.forEach(function (placeType) { - if (place.parent[placeType] && place.parent[placeType].length > 0 && place.parent[placeType][0]) { - place.parent[placeType + '_id'] = [ makeNewId(placeType, place.parent[placeType + '_id']) ]; + if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) { + place[placeType + '_id'] = [ makeNewId(placeType, place[placeType + '_id']) ]; } }); } diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js index 9ad603f8..95dc1387 100644 --- a/test/unit/middleware/normalizeParentIds.js +++ b/test/unit/middleware/normalizeParentIds.js @@ -8,30 +8,33 @@ module.exports.tests.interface = function(test, common) { var input = { data: [{ 'parent': { - 'country': ['United States'], + 'country': ['United States'], // these shouldn't change 'country_id': ['85633793'], - 'country_a': ['USA'], - 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['foobar'], - 'macroregion_a': ['MacroRegion Abbreviation'], - 'region': ['New York'], - 'region_id': ['85688543'], - 'region_a': ['NY'], - 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['~~~~~'], - 'macrocounty_a': ['MacroCounty Abbreviation'], - 'county': ['Kings County'], - 'county_id': ['102082361'], - 'county_a': [null], - 'localadmin': ['Brooklyn'], - 'localadmin_id': ['404521211'], - 'localadmin_a': [null], - 'locality': ['Some Locality'], - 'locality_id': ['85977539'], - 'locality_a': [null], - 'neighbourhood': [], - 'neighbourhood_id': [] - } + 'country_a': ['USA'] + }, + 'country': ['United States'], + 'country_id': ['85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] }] }; @@ -39,29 +42,32 @@ module.exports.tests.interface = function(test, common) { data: [{ 'parent': { 'country': ['United States'], - 'country_id': ['whosonfirst:country:85633793'], - 'country_a': ['USA'], - 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['whosonfirst:macroregion:foobar'], - 'macroregion_a': ['MacroRegion Abbreviation'], - 'region': ['New York'], - 'region_id': ['whosonfirst:region:85688543'], - 'region_a': ['NY'], - 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], - 'macrocounty_a': ['MacroCounty Abbreviation'], - 'county': ['Kings County'], - 'county_id': ['whosonfirst:county:102082361'], - 'county_a': [null], - 'localadmin': ['Brooklyn'], - 'localadmin_id': ['whosonfirst:localadmin:404521211'], - 'localadmin_a': [null], - 'locality': ['Some Locality'], - 'locality_id': ['whosonfirst:locality:85977539'], - 'locality_a': [null], - 'neighbourhood': [], - 'neighbourhood_id': [] - } + 'country_id': ['85633793'], + 'country_a': ['USA'] + }, + 'country': ['United States'], + 'country_id': ['whosonfirst:country:85633793'], + 'country_a': ['USA'], + 'macroregion': ['MacroRegion Name'], + 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_a': ['MacroRegion Abbreviation'], + 'region': ['New York'], + 'region_id': ['whosonfirst:region:85688543'], + 'region_a': ['NY'], + 'macrocounty': ['MacroCounty Name'], + 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_a': ['MacroCounty Abbreviation'], + 'county': ['Kings County'], + 'county_id': ['whosonfirst:county:102082361'], + 'county_a': [null], + 'localadmin': ['Brooklyn'], + 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_a': [null], + 'locality': ['Some Locality'], + 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_a': [null], + 'neighbourhood': [], + 'neighbourhood_id': [] }] }; From 147c849b58ced5a3c1044f115ede3e7f0b00eca6 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 7 Apr 2016 16:41:08 -0400 Subject: [PATCH 105/106] change admin property names to _gid from _id --- helper/geojsonify.js | 21 +++++++----- helper/placeTypes.js | 11 ++++++ middleware/normalizeParentIds.js | 14 ++------ middleware/renamePlacenames.js | 40 ++-------------------- test/unit/helper/geojsonify.js | 38 ++++++++++++-------- test/unit/middleware/normalizeParentIds.js | 32 ++++++++--------- 6 files changed, 67 insertions(+), 89 deletions(-) create mode 100644 helper/placeTypes.js diff --git a/helper/geojsonify.js b/helper/geojsonify.js index a6e082eb..a2dc3e2b 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -4,6 +4,7 @@ var GeoJSON = require('geojson'), labelGenerator = require('./labelGenerator'), logger = require('pelias-logger').get('api'), type_mapping = require('./type_mapping'), + Document = require('pelias-model').Document, _ = require('lodash'); // Properties to be copied @@ -14,28 +15,29 @@ var DETAILS_PROPS = [ 'confidence', 'distance', 'country', - 'country_id', + 'country_gid', 'country_a', 'macroregion', - 'macroregion_id', + 'macroregion_gid', 'macroregion_a', 'region', - 'region_id', + 'region_gid', 'region_a', 'macrocounty', - 'macrocounty_id', + 'macrocounty_gid', 'macrocounty_a', 'county', - 'county_id', + 'county_gid', 'county_a', 'localadmin', - 'localadmin_id', + 'localadmin_gid', 'localadmin_a', 'locality', - 'locality_id', + 'locality_gid', 'locality_a', 'neighbourhood', - 'neighbourhood_id' + 'neighbourhood_gid', + 'bounding_box' ]; @@ -235,7 +237,8 @@ function copyProperties( source, props, dst ) { * @param {object} src */ function makeGid(src) { - return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id; + var doc = new Document(lookupSource(src), lookupLayer(src), src._id); + return doc.getGid(); } /** diff --git a/helper/placeTypes.js b/helper/placeTypes.js new file mode 100644 index 00000000..76538982 --- /dev/null +++ b/helper/placeTypes.js @@ -0,0 +1,11 @@ +module.exports = [ + 'country', + 'macroregion', + 'region', + 'macrocounty', + 'county', + 'localadmin', + 'locality', + 'borough', + 'neighbourhood' +]; diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index fe512b61..e0e499e2 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -1,17 +1,7 @@ var logger = require('pelias-logger').get('api'); var Document = require('pelias-model').Document; -var placeTypes = [ - 'country', - 'macroregion', - 'region', - 'macrocounty', - 'county', - 'localadmin', - 'locality', - 'neighbourhood', - 'borough' -]; +var placeTypes = require('../helper/placeTypes'); /** * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint. @@ -42,7 +32,7 @@ function normalizeParentIds(place) { if (place) { placeTypes.forEach(function (placeType) { if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) { - place[placeType + '_id'] = [ makeNewId(placeType, place[placeType + '_id']) ]; + place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ]; } }); } diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 152ad404..d89a43ad 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -1,16 +1,6 @@ var _ = require('lodash'); -/** - - P is a preferred English name - - Q is a preferred name (in other languages) - - V is a well-known (but unofficial) variant for the place - (e.g. "New York City" for New York) - - S is either a synonym or a colloquial name for the place - (e.g. "Big Apple" for New York), or a version of the name which - is stripped of accent characters. - - A is an abbreviation or code for the place (e.g. "NYC" for New - York) - */ +var PARENT_PROPS = require('../helper/placeTypes'); var ADDRESS_PROPS = { 'number': 'housenumber', @@ -18,32 +8,6 @@ var ADDRESS_PROPS = { 'street': 'street' }; -var PARENT_PROPS = [ - 'country', - 'country_id', - 'country_a', - 'macroregion', - 'macroregion_id', - 'macroregion_a', - 'region', - 'region_id', - 'region_a', - 'macrocounty', - 'macrocounty_id', - 'macrocounty_a', - 'county', - 'county_id', - 'county_a', - 'localadmin', - 'localadmin_id', - 'localadmin_a', - 'locality', - 'locality_id', - 'locality_a', - 'neighbourhood', - 'neighbourhood_id' -]; - function setup() { return renamePlacenames; @@ -74,6 +38,8 @@ function renameOneRecord(place) { if (place.parent) { PARENT_PROPS.forEach(function (prop) { place[prop] = place.parent[prop]; + place[prop + '_a'] = place.parent[prop + '_a']; + place[prop + '_gid'] = place.parent[prop + '_id']; }); } diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 132f1e97..f5c89329 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) { var earth = [{ '_type': 'geoname', '_id': '6295630', + 'source': 'whosonfirst', + 'layer': 'continent', 'name': { 'default': 'Earth' }, @@ -240,7 +242,7 @@ module.exports.tests.search = function(test, common) { 'country': [ 'United States' ], - 'country_id': [ + 'country_gid': [ '85633793' ], 'country_a': [ @@ -249,7 +251,7 @@ module.exports.tests.search = function(test, common) { 'macroregion': [ 'MacroRegion Name' ], - 'macroregion_id': [ + 'macroregion_gid': [ 'MacroRegion Id' ], 'macroregion_a': [ @@ -258,7 +260,7 @@ module.exports.tests.search = function(test, common) { 'region': [ 'New York' ], - 'region_id': [ + 'region_gid': [ '85688543' ], 'region_a': [ @@ -267,7 +269,7 @@ module.exports.tests.search = function(test, common) { 'macrocounty': [ 'MacroCounty Name' ], - 'macrocounty_id': [ + 'macrocounty_gid': [ 'MacroCounty Id' ], 'macrocounty_a': [ @@ -276,7 +278,7 @@ module.exports.tests.search = function(test, common) { 'county': [ 'Kings County' ], - 'county_id': [ + 'county_gid': [ '102082361' ], 'county_a': [ @@ -285,20 +287,20 @@ module.exports.tests.search = function(test, common) { 'localadmin': [ 'Brooklyn' ], - 'localadmin_id': [ + 'localadmin_gid': [ '404521211' ], 'localadmin_a': [ null ], - 'locality_id': [ + 'locality_gid': [ '85977539' ], 'locality_a': [ null ], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] } ]; @@ -316,23 +318,29 @@ module.exports.tests.search = function(test, common) { 'name': 'East New York', 'confidence': 0.888, 'country': 'United States', - 'country_id': '85633793', + 'country_gid': '85633793', 'country_a': 'USA', 'macroregion': 'MacroRegion Name', - 'macroregion_id': 'MacroRegion Id', + 'macroregion_gid': 'MacroRegion Id', 'macroregion_a': 'MacroRegion Abbreviation', 'region': 'New York', - 'region_id': '85688543', + 'region_gid': '85688543', 'region_a': 'NY', 'macrocounty': 'MacroCounty Name', - 'macrocounty_id': 'MacroCounty Id', + 'macrocounty_gid': 'MacroCounty Id', 'macrocounty_a': 'MacroCounty Abbreviation', 'county': 'Kings County', - 'county_id': '102082361', + 'county_gid': '102082361', 'localadmin': 'Brooklyn', - 'localadmin_id': '404521211', + 'localadmin_gid': '404521211', 'locality': 'New York', - 'locality_id': '85977539', + 'locality_gid': '85977539', + 'bounding_box': { + 'min_lat': 40.6514712164, + 'max_lat': 40.6737320588, + 'min_lon': -73.8967895508, + 'max_lon': -73.8665771484 + }, 'label': 'East New York, Brooklyn, NY, USA' }, 'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588], diff --git a/test/unit/middleware/normalizeParentIds.js b/test/unit/middleware/normalizeParentIds.js index 95dc1387..472df0e3 100644 --- a/test/unit/middleware/normalizeParentIds.js +++ b/test/unit/middleware/normalizeParentIds.js @@ -13,28 +13,28 @@ module.exports.tests.interface = function(test, common) { 'country_a': ['USA'] }, 'country': ['United States'], - 'country_id': ['85633793'], + 'country_gid': ['85633793'], 'country_a': ['USA'], 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['foobar'], + 'macroregion_gid': ['foobar'], 'macroregion_a': ['MacroRegion Abbreviation'], 'region': ['New York'], - 'region_id': ['85688543'], + 'region_gid': ['85688543'], 'region_a': ['NY'], 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['~~~~~'], + 'macrocounty_gid': ['~~~~~'], 'macrocounty_a': ['MacroCounty Abbreviation'], 'county': ['Kings County'], - 'county_id': ['102082361'], + 'county_gid': ['102082361'], 'county_a': [null], 'localadmin': ['Brooklyn'], - 'localadmin_id': ['404521211'], + 'localadmin_gid': ['404521211'], 'localadmin_a': [null], 'locality': ['Some Locality'], - 'locality_id': ['85977539'], + 'locality_gid': ['85977539'], 'locality_a': [null], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] }] }; @@ -46,28 +46,28 @@ module.exports.tests.interface = function(test, common) { 'country_a': ['USA'] }, 'country': ['United States'], - 'country_id': ['whosonfirst:country:85633793'], + 'country_gid': ['whosonfirst:country:85633793'], 'country_a': ['USA'], 'macroregion': ['MacroRegion Name'], - 'macroregion_id': ['whosonfirst:macroregion:foobar'], + 'macroregion_gid': ['whosonfirst:macroregion:foobar'], 'macroregion_a': ['MacroRegion Abbreviation'], 'region': ['New York'], - 'region_id': ['whosonfirst:region:85688543'], + 'region_gid': ['whosonfirst:region:85688543'], 'region_a': ['NY'], 'macrocounty': ['MacroCounty Name'], - 'macrocounty_id': ['whosonfirst:macrocounty:~~~~~'], + 'macrocounty_gid': ['whosonfirst:macrocounty:~~~~~'], 'macrocounty_a': ['MacroCounty Abbreviation'], 'county': ['Kings County'], - 'county_id': ['whosonfirst:county:102082361'], + 'county_gid': ['whosonfirst:county:102082361'], 'county_a': [null], 'localadmin': ['Brooklyn'], - 'localadmin_id': ['whosonfirst:localadmin:404521211'], + 'localadmin_gid': ['whosonfirst:localadmin:404521211'], 'localadmin_a': [null], 'locality': ['Some Locality'], - 'locality_id': ['whosonfirst:locality:85977539'], + 'locality_gid': ['whosonfirst:locality:85977539'], 'locality_a': [null], 'neighbourhood': [], - 'neighbourhood_id': [] + 'neighbourhood_gid': [] }] }; From 8a15234e56529fbb5d7d63df5e67605f25c4efec Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 7 Apr 2016 17:49:47 -0400 Subject: [PATCH 106/106] Fix failing test --- test/unit/helper/geojsonify.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index f5c89329..ea784b62 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -335,12 +335,6 @@ module.exports.tests.search = function(test, common) { 'localadmin_gid': '404521211', 'locality': 'New York', 'locality_gid': '85977539', - 'bounding_box': { - 'min_lat': 40.6514712164, - 'max_lat': 40.6737320588, - 'min_lon': -73.8967895508, - 'max_lon': -73.8665771484 - }, 'label': 'East New York, Brooklyn, NY, USA' }, 'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],