From a299106a2c550df9196313b271ceb20b119c132c Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Tue, 29 Sep 2015 17:39:57 -0400 Subject: [PATCH 01/17] circle.yml: automate api deployments for prod/prod_build --- circle.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 circle.yml diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..a46b8e50 --- /dev/null +++ b/circle.yml @@ -0,0 +1,19 @@ +machine: + ruby: + version: 2.1.2 + node: + version: 0.12.2 + +dependencies: + pre: + - echo "{\"mapzen\":{\"api_key\":{\"pelias.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json + +deployment: + prod: + branch: production + commands: + - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install + - cd acceptance-tests && npm test -- -e prod + - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install + - cd pelias-deploy && bundle exec rake deploy:api[prod] + - cd pelias-deploy && bundle exec rake deploy:api[prod_build] From b1f615479c968efde3dc633fdca808cb2d6f92f5 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 12 Oct 2015 16:51:27 -0400 Subject: [PATCH 02/17] Replace tap-spec module with tap-dot --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index da77a3df..5a4ea6e5 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "nsp": "^0.3.0", "precommit-hook": "^1.0.7", "proxyquire": "^1.4.0", - "tap-spec": "^0.2.0", + "tap-dot": "^1.0.0", "tape": "^2.13.4" } } From 098c3c74abcbae7a54cf7f4edb6e9932626be2c2 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 12 Oct 2015 16:51:40 -0400 Subject: [PATCH 03/17] Use tap-dot when running tests --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5a4ea6e5..be08ea46 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "start": "node index.js", "test": "npm run unit", - "unit": "node test/unit/run.js | tap-spec", + "unit": "node test/unit/run.js | tap-dot", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", From f5d823ddc23cf3a82db5db871d8f2cd63e47c4d3 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Tue, 13 Oct 2015 09:29:37 -0400 Subject: [PATCH 04/17] fix circle.yml typo --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index a46b8e50..3b287f46 100644 --- a/circle.yml +++ b/circle.yml @@ -6,7 +6,7 @@ machine: dependencies: pre: - - echo "{\"mapzen\":{\"api_key\":{\"pelias.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json + - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json deployment: prod: From 8f1c8f4d890e7bd76a6897b12cbc75f6452c6ee0 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Oct 2015 15:51:00 -0400 Subject: [PATCH 05/17] Add helpful comments in various query parser locations --- helper/query_parser.js | 4 ++++ helper/types.js | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/helper/query_parser.js b/helper/query_parser.js index cb5dc7a7..a561aef9 100644 --- a/helper/query_parser.js +++ b/helper/query_parser.js @@ -8,6 +8,10 @@ var logger = require('pelias-logger').get('api'); module.exports = {}; +/* + * For performance, and to prefer POI and admin records, express a preference + * to only search coarse layers on very short text inputs. + */ module.exports.get_layers = function get_layers(query) { if (query.length <= 3 ) { // no address parsing required diff --git a/helper/types.js b/helper/types.js index 610d1aba..bea2db6f 100644 --- a/helper/types.js +++ b/helper/types.js @@ -2,12 +2,14 @@ var type_mapping = require( '../helper/type_mapping' ); var _ = require('lodash'); /** - * Combine all types and determine the unique subset + * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched + * This method decides how to combine all the preferences. * * @param {Array} clean_types * @returns {Array} */ module.exports = function calculate_types(clean_types) { + //Check that at least one preference of types is defined if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_address_parser)) { throw new Error('clean_types should not be null or undefined'); } From 59b70f7c7eb22b91829f01e0141a83090ed229d3 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Oct 2015 15:55:02 -0400 Subject: [PATCH 06/17] Rename helper/query_parser to helper/text_parser --- helper/{query_parser.js => text_parser.js} | 0 sanitiser/_text.js | 6 +++--- test/unit/helper/{query_parser.js => text_parser.js} | 2 +- test/unit/query/autocomplete.js | 2 +- test/unit/query/search.js | 2 +- test/unit/run.js | 2 +- test/unit/sanitiser/search.js | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename helper/{query_parser.js => text_parser.js} (100%) rename test/unit/helper/{query_parser.js => text_parser.js} (98%) diff --git a/helper/query_parser.js b/helper/text_parser.js similarity index 100% rename from helper/query_parser.js rename to helper/text_parser.js diff --git a/sanitiser/_text.js b/sanitiser/_text.js index d0cc0986..0aa5c26c 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -1,5 +1,5 @@ var check = require('check-types'), - query_parser = require('../helper/query_parser'); + text_parser = require('../helper/text_parser'); // validate texts, convert types and apply defaults function sanitize( raw, clean ){ @@ -19,14 +19,14 @@ function sanitize( raw, clean ){ clean.text = raw.text; // parse text with query parser - var parsed_text = query_parser.get_parsed_address(clean.text); + var parsed_text = text_parser.get_parsed_address(clean.text); if (check.assigned(parsed_text)) { clean.parsed_text = parsed_text; } // try to set layers from query parser results clean.types = clean.layers || {}; - clean.types.from_address_parsing = query_parser.get_layers(clean.text); + clean.types.from_address_parsing = text_parser.get_layers(clean.text); } return messages; diff --git a/test/unit/helper/query_parser.js b/test/unit/helper/text_parser.js similarity index 98% rename from test/unit/helper/query_parser.js rename to test/unit/helper/text_parser.js index e7771ca4..a4ce64e3 100644 --- a/test/unit/helper/query_parser.js +++ b/test/unit/helper/text_parser.js @@ -1,4 +1,4 @@ -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); var type_mapping = require('../../../helper/type_mapping'); var layers_map = type_mapping.layer_with_aliases_to_type; diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js index ecdf9553..c3f620c0 100644 --- a/test/unit/query/autocomplete.js +++ b/test/unit/query/autocomplete.js @@ -1,6 +1,6 @@ var generate = require('../../../query/autocomplete'); -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); module.exports.tests = {}; diff --git a/test/unit/query/search.js b/test/unit/query/search.js index 3a11e054..15527222 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -1,5 +1,5 @@ var generate = require('../../../query/search'); -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); module.exports.tests = {}; diff --git a/test/unit/run.js b/test/unit/run.js index 7e0fe272..fb8b1728 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -9,7 +9,7 @@ var tests = [ require('./helper/geojsonify'), require('./helper/labelGenerator'), require('./helper/labelSchema'), - require('./helper/query_parser'), + require('./helper/text_parser'), require('./helper/type_mapping'), require('./helper/types'), require('./middleware/confidenceScore'), diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index 4f4e2880..b00160dd 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -1,6 +1,6 @@ var extend = require('extend'), search = require('../../../sanitiser/search'), - parser = require('../../../helper/query_parser'), + parser = require('../../../helper/text_parser'), sanitize = search.sanitize, middleware = search.middleware, defaultError = 'invalid param \'text\': text length, must be >0'; From b188a2046c876e124d8f6fda4fc7ce16b1a04a8c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Oct 2015 16:03:00 -0400 Subject: [PATCH 07/17] Fix mismatching name of types preference parameter The parameter in clean.types being set by `helpers/text_parser.js` was "from_address_parsing", but the code in `helper/types.js` was expecting "from_address_parser". This commit makes both use "from_address_parser" and adds a test. --- sanitiser/_text.js | 2 +- test/unit/run.js | 1 + test/unit/sanitiser/_text.js | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/unit/sanitiser/_text.js diff --git a/sanitiser/_text.js b/sanitiser/_text.js index 0aa5c26c..e8b055f0 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -26,7 +26,7 @@ function sanitize( raw, clean ){ // try to set layers from query parser results clean.types = clean.layers || {}; - clean.types.from_address_parsing = text_parser.get_layers(clean.text); + clean.types.from_address_parser = text_parser.get_layers(clean.text); } return messages; diff --git a/test/unit/run.js b/test/unit/run.js index fb8b1728..b79f9aca 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -29,6 +29,7 @@ var tests = [ require('./sanitiser/_single_scalar_parameters'), require('./sanitiser/_size'), require('./sanitiser/_sources'), + require('./sanitiser/_text'), require('./sanitiser/autocomplete'), require('./sanitiser/place'), require('./sanitiser/reverse'), diff --git a/test/unit/sanitiser/_text.js b/test/unit/sanitiser/_text.js new file mode 100644 index 00000000..3ccfbbb2 --- /dev/null +++ b/test/unit/sanitiser/_text.js @@ -0,0 +1,32 @@ +var sanitiser = require('../../../sanitiser/_text'); +var type_mapping = require('../../../helper/type_mapping'); + +module.exports.tests = {}; + +module.exports.tests.text_parser = function(test, common) { + test('short input text has admin layers set ', function(t) { + var raw = { + text: 'emp' //start of empire state building + }; + var clean = { + }; + + var messages = sanitiser(raw, clean); + + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + t.equal(clean.types.from_address_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred'); + + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANITISER _text: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; From db895c05b71650971a98540a62908239b263312b Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Oct 2015 16:08:05 -0400 Subject: [PATCH 08/17] Rename `from_address_parser` key to `from_text_parser` --- helper/types.js | 6 +++--- sanitiser/_text.js | 2 +- test/unit/helper/types.js | 4 ++-- test/unit/sanitiser/_text.js | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/helper/types.js b/helper/types.js index bea2db6f..d9dec59d 100644 --- a/helper/types.js +++ b/helper/types.js @@ -10,7 +10,7 @@ var _ = require('lodash'); */ module.exports = function calculate_types(clean_types) { //Check that at least one preference of types is defined - if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_address_parser)) { + if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) { throw new Error('clean_types should not be null or undefined'); } @@ -35,8 +35,8 @@ module.exports = function calculate_types(clean_types) { * Type restrictions requested by the address parser should only be used * if both the source and layers parameters are empty, so do this last */ - if (clean_types.from_address_parser) { - return clean_types.from_address_parser; + if (clean_types.from_text_parser) { + return clean_types.from_text_parser; } throw new Error('no types specified'); diff --git a/sanitiser/_text.js b/sanitiser/_text.js index e8b055f0..824c7a87 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -26,7 +26,7 @@ function sanitize( raw, clean ){ // try to set layers from query parser results clean.types = clean.layers || {}; - clean.types.from_address_parser = text_parser.get_layers(clean.text); + clean.types.from_text_parser = text_parser.get_layers(clean.text); } return messages; diff --git a/test/unit/helper/types.js b/test/unit/helper/types.js index 4638ff12..92848d46 100644 --- a/test/unit/helper/types.js +++ b/test/unit/helper/types.js @@ -33,7 +33,7 @@ module.exports.tests.no_cleaned_types = function(test, common) { module.exports.tests.address_parser = function(test, common) { test('address parser specifies only admin layers', function(t) { var cleaned_types = { - from_address_parser: ['admin0'] // simplified return value from address parser + from_text_parser: ['admin0'] // simplified return value from address parser }; var actual = types(cleaned_types); var expected = ['admin0']; // simplified expected value for all admin layers @@ -58,7 +58,7 @@ module.exports.tests.layers_parameter_and_address_parser = function(test, common test('layers parameter and address parser present', function(t) { var cleaned_types = { from_layers: ['geoname'], - from_address_parser: ['admin0'] // simplified return value from address parse + from_text_parser: ['admin0'] // simplified return value from address parse }; var actual = types(cleaned_types); var expected = ['geoname']; diff --git a/test/unit/sanitiser/_text.js b/test/unit/sanitiser/_text.js index 3ccfbbb2..8bc0710f 100644 --- a/test/unit/sanitiser/_text.js +++ b/test/unit/sanitiser/_text.js @@ -15,7 +15,7 @@ module.exports.tests.text_parser = function(test, common) { t.deepEquals(messages.errors, [], 'no errors'); t.deepEquals(messages.warnings, [], 'no warnings'); - t.equal(clean.types.from_address_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred'); + t.equal(clean.types.from_text_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred'); t.end(); }); From 84bae8226b231e5cd6f61c26f7672da6826886a0 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Thu, 15 Oct 2015 09:53:20 -0400 Subject: [PATCH 09/17] should only need search api key --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 3b287f46..4be6b301 100644 --- a/circle.yml +++ b/circle.yml @@ -6,7 +6,7 @@ machine: dependencies: pre: - - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json + - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\"}}}" >~/pelias.json deployment: prod: From 49d37eafed15072d6e35e5ce42bcc4e79e3ba4a4 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 20 Oct 2015 17:10:01 +0200 Subject: [PATCH 10/17] one config per query --- query/autocomplete.js | 2 +- .../{defaults.js => autocomplete_defaults.js} | 0 query/reverse.js | 2 +- query/reverse_defaults.js | 95 +++++++++++++++++++ query/search.js | 2 +- query/search_defaults.js | 95 +++++++++++++++++++ sanitiser/_geo_reverse.js | 2 +- test/unit/query/autocomplete_defaults.js | 22 +++++ test/unit/query/reverse_defaults.js | 22 +++++ .../query/{defaults.js => search_defaults.js} | 4 +- test/unit/run.js | 4 +- test/unit/sanitiser/_geo_reverse.js | 2 +- test/unit/sanitiser/reverse.js | 2 +- 13 files changed, 245 insertions(+), 9 deletions(-) rename query/{defaults.js => autocomplete_defaults.js} (100%) create mode 100644 query/reverse_defaults.js create mode 100644 query/search_defaults.js create mode 100644 test/unit/query/autocomplete_defaults.js create mode 100644 test/unit/query/reverse_defaults.js rename test/unit/query/{defaults.js => search_defaults.js} (77%) diff --git a/query/autocomplete.js b/query/autocomplete.js index ad64413a..0e9b9d68 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -1,6 +1,6 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./autocomplete_defaults'), check = require('check-types'); //------------------------------ diff --git a/query/defaults.js b/query/autocomplete_defaults.js similarity index 100% rename from query/defaults.js rename to query/autocomplete_defaults.js diff --git a/query/reverse.js b/query/reverse.js index fd84b427..b48597c1 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,5 +1,5 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./reverse_defaults'), check = require('check-types'); //------------------------------ diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js new file mode 100644 index 00000000..b61e84ac --- /dev/null +++ b/query/reverse_defaults.js @@ -0,0 +1,95 @@ + +var peliasQuery = require('pelias-query'), + extend = require('extend'); + +module.exports = extend( false, peliasQuery.defaults, { + + 'size': 10, + 'track_scores': true, + + 'centroid:field': 'center_point', + + 'sort:distance:order': 'asc', + 'sort:distance:distance_type': 'plane', + + 'boundary:circle:radius': '50km', + 'boundary:circle:distance_type': 'plane', + 'boundary:circle:optimize_bbox': 'indexed', + 'boundary:circle:_cache': true, + + 'boundary:rect:type': 'indexed', + 'boundary:rect:_cache': true, + + 'ngram:analyzer': 'peliasOneEdgeGram', + 'ngram:field': 'name.default', + 'ngram:boost': 1, + + 'phrase:analyzer': 'peliasPhrase', + 'phrase:field': 'phrase.default', + 'phrase:boost': 1, + 'phrase:slop': 2, + + 'focus:function': 'linear', + 'focus:offset': '1km', + 'focus:scale': '50km', + 'focus:decay': 0.5, + 'focus:weight': 2, + + 'function_score:score_mode': 'avg', + 'function_score:boost_mode': 'replace', + + 'address:housenumber:analyzer': 'standard', + 'address:housenumber:field': 'address.number', + 'address:housenumber:boost': 2, + + 'address:street:analyzer': 'standard', + 'address:street:field': 'address.street', + 'address:street:boost': 5, + + 'address:postcode:analyzer': 'standard', + 'address:postcode:field': 'address.zip', + 'address:postcode:boost': 3, + + 'admin:alpha3:analyzer': 'standard', + 'admin:alpha3:field': 'alpha3', + 'admin:alpha3:boost': 5, + + 'admin:admin0:analyzer': 'peliasAdmin', + 'admin:admin0:field': 'admin0', + 'admin:admin0:boost': 4, + + 'admin:admin1:analyzer': 'peliasAdmin', + 'admin:admin1:field': 'admin1', + 'admin:admin1:boost': 3, + + 'admin:admin1_abbr:analyzer': 'peliasAdmin', + 'admin:admin1_abbr:field': 'admin1_abbr', + 'admin:admin1_abbr:boost': 3, + + 'admin:admin2:analyzer': 'peliasAdmin', + 'admin:admin2:field': 'admin2', + 'admin:admin2:boost': 2, + + 'admin:local_admin:analyzer': 'peliasAdmin', + 'admin:local_admin:field': 'local_admin', + 'admin:local_admin:boost': 1, + + 'admin:locality:analyzer': 'peliasAdmin', + 'admin:locality:field': 'locality', + 'admin:locality:boost': 1, + + 'admin:neighborhood:analyzer': 'peliasAdmin', + 'admin:neighborhood:field': 'neighborhood', + 'admin:neighborhood:boost': 1, + + 'popularity:field': 'popularity', + 'popularity:modifier': 'log1p', + 'popularity:max_boost': 20, + 'popularity:weight': 1, + + 'population:field': 'population', + 'population:modifier': 'log1p', + 'population:max_boost': 20, + 'population:weight': 2 + +}); diff --git a/query/search.js b/query/search.js index a30aa2fe..da7773a9 100644 --- a/query/search.js +++ b/query/search.js @@ -1,5 +1,5 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./search_defaults'), textParser = require('./text_parser'), check = require('check-types'), geolib = require('geolib'); diff --git a/query/search_defaults.js b/query/search_defaults.js new file mode 100644 index 00000000..b61e84ac --- /dev/null +++ b/query/search_defaults.js @@ -0,0 +1,95 @@ + +var peliasQuery = require('pelias-query'), + extend = require('extend'); + +module.exports = extend( false, peliasQuery.defaults, { + + 'size': 10, + 'track_scores': true, + + 'centroid:field': 'center_point', + + 'sort:distance:order': 'asc', + 'sort:distance:distance_type': 'plane', + + 'boundary:circle:radius': '50km', + 'boundary:circle:distance_type': 'plane', + 'boundary:circle:optimize_bbox': 'indexed', + 'boundary:circle:_cache': true, + + 'boundary:rect:type': 'indexed', + 'boundary:rect:_cache': true, + + 'ngram:analyzer': 'peliasOneEdgeGram', + 'ngram:field': 'name.default', + 'ngram:boost': 1, + + 'phrase:analyzer': 'peliasPhrase', + 'phrase:field': 'phrase.default', + 'phrase:boost': 1, + 'phrase:slop': 2, + + 'focus:function': 'linear', + 'focus:offset': '1km', + 'focus:scale': '50km', + 'focus:decay': 0.5, + 'focus:weight': 2, + + 'function_score:score_mode': 'avg', + 'function_score:boost_mode': 'replace', + + 'address:housenumber:analyzer': 'standard', + 'address:housenumber:field': 'address.number', + 'address:housenumber:boost': 2, + + 'address:street:analyzer': 'standard', + 'address:street:field': 'address.street', + 'address:street:boost': 5, + + 'address:postcode:analyzer': 'standard', + 'address:postcode:field': 'address.zip', + 'address:postcode:boost': 3, + + 'admin:alpha3:analyzer': 'standard', + 'admin:alpha3:field': 'alpha3', + 'admin:alpha3:boost': 5, + + 'admin:admin0:analyzer': 'peliasAdmin', + 'admin:admin0:field': 'admin0', + 'admin:admin0:boost': 4, + + 'admin:admin1:analyzer': 'peliasAdmin', + 'admin:admin1:field': 'admin1', + 'admin:admin1:boost': 3, + + 'admin:admin1_abbr:analyzer': 'peliasAdmin', + 'admin:admin1_abbr:field': 'admin1_abbr', + 'admin:admin1_abbr:boost': 3, + + 'admin:admin2:analyzer': 'peliasAdmin', + 'admin:admin2:field': 'admin2', + 'admin:admin2:boost': 2, + + 'admin:local_admin:analyzer': 'peliasAdmin', + 'admin:local_admin:field': 'local_admin', + 'admin:local_admin:boost': 1, + + 'admin:locality:analyzer': 'peliasAdmin', + 'admin:locality:field': 'locality', + 'admin:locality:boost': 1, + + 'admin:neighborhood:analyzer': 'peliasAdmin', + 'admin:neighborhood:field': 'neighborhood', + 'admin:neighborhood:boost': 1, + + 'popularity:field': 'popularity', + 'popularity:modifier': 'log1p', + 'popularity:max_boost': 20, + 'popularity:weight': 1, + + 'population:field': 'population', + 'population:modifier': 'log1p', + 'population:max_boost': 20, + 'population:weight': 2 + +}); diff --git a/sanitiser/_geo_reverse.js b/sanitiser/_geo_reverse.js index c15af48b..52f2980f 100644 --- a/sanitiser/_geo_reverse.js +++ b/sanitiser/_geo_reverse.js @@ -1,7 +1,7 @@ var geo_common = require ('./_geo_common'); var _ = require('lodash'); -var defaults = require('../query/defaults'); +var defaults = require('../query/reverse_defaults'); var LAT_LON_IS_REQUIRED = true, CIRCLE_IS_REQUIRED = false; diff --git a/test/unit/query/autocomplete_defaults.js b/test/unit/query/autocomplete_defaults.js new file mode 100644 index 00000000..7ccb17a0 --- /dev/null +++ b/test/unit/query/autocomplete_defaults.js @@ -0,0 +1,22 @@ + +var defaults = require('../../../query/autocomplete_defaults'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof defaults, 'object', 'defaults defined'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('autocomplete defaults ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/reverse_defaults.js b/test/unit/query/reverse_defaults.js new file mode 100644 index 00000000..48a9805f --- /dev/null +++ b/test/unit/query/reverse_defaults.js @@ -0,0 +1,22 @@ + +var defaults = require('../../../query/reverse_defaults'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof defaults, 'object', 'defaults defined'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('reverse defaults ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/defaults.js b/test/unit/query/search_defaults.js similarity index 77% rename from test/unit/query/defaults.js rename to test/unit/query/search_defaults.js index d6fcc720..54b116b7 100644 --- a/test/unit/query/defaults.js +++ b/test/unit/query/search_defaults.js @@ -1,5 +1,5 @@ -var defaults = require('../../../query/defaults'); +var defaults = require('../../../query/search_defaults'); module.exports.tests = {}; @@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) { module.exports.all = function (tape, common) { function test(name, testFunction) { - return tape('query defaults ' + name, testFunction); + return tape('search defaults ' + name, testFunction); } for( var testCase in module.exports.tests ){ diff --git a/test/unit/run.js b/test/unit/run.js index b79f9aca..76a3a437 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -16,7 +16,9 @@ var tests = [ require('./middleware/confidenceScoreReverse'), require('./middleware/distance'), require('./query/autocomplete'), - require('./query/defaults'), + require('./query/autocomplete_defaults'), + require('./query/search_defaults'), + require('./query/reverse_defaults'), require('./query/reverse'), require('./query/search'), require('./sanitiser/_boundary_country'), diff --git a/test/unit/sanitiser/_geo_reverse.js b/test/unit/sanitiser/_geo_reverse.js index e03e7ddd..e7c62e07 100644 --- a/test/unit/sanitiser/_geo_reverse.js +++ b/test/unit/sanitiser/_geo_reverse.js @@ -1,5 +1,5 @@ var sanitize = require('../../../sanitiser/_geo_reverse'); -var defaults = require('../../../query/defaults'); +var defaults = require('../../../query/reverse_defaults'); module.exports.tests = {}; diff --git a/test/unit/sanitiser/reverse.js b/test/unit/sanitiser/reverse.js index b92896e7..4e727b5b 100644 --- a/test/unit/sanitiser/reverse.js +++ b/test/unit/sanitiser/reverse.js @@ -4,7 +4,7 @@ var reverse = require('../../../sanitiser/reverse'), sanitize = reverse.sanitize, middleware = reverse.middleware, - defaults = require('../../../query/defaults'), + defaults = require('../../../query/reverse_defaults'), defaultError = 'missing param \'lat\'', defaultClean = { 'point.lat': 0, 'point.lon': 0, From 82509890254d02f1a33b90fad6d53723e93bfb20 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Wed, 28 Oct 2015 07:26:36 -0400 Subject: [PATCH 11/17] Revert "should only need search api key" This reverts commit 84bae8226b231e5cd6f61c26f7672da6826886a0. --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 4be6b301..3b287f46 100644 --- a/circle.yml +++ b/circle.yml @@ -6,7 +6,7 @@ machine: dependencies: pre: - - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\"}}}" >~/pelias.json + - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json deployment: prod: From a8245efc3350ed00d8f7c6b51032568705233311 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Wed, 28 Oct 2015 07:33:37 -0400 Subject: [PATCH 12/17] deploy master to dev --- circle.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/circle.yml b/circle.yml index 3b287f46..d5df506a 100644 --- a/circle.yml +++ b/circle.yml @@ -17,3 +17,10 @@ deployment: - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[prod] - cd pelias-deploy && bundle exec rake deploy:api[prod_build] + dev: + branch: master + commands: + - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install + - cd acceptance-tests && npm test -- -e prod + - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install + - cd pelias-deploy && bundle exec rake deploy:api[dev] From a9fe638dfb431521f887e0f70bbdcfea1ebaed90 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Wed, 28 Oct 2015 07:35:17 -0400 Subject: [PATCH 13/17] acceptance tests run against dev --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index d5df506a..0fd2eaf7 100644 --- a/circle.yml +++ b/circle.yml @@ -21,6 +21,6 @@ deployment: branch: master commands: - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install - - cd acceptance-tests && npm test -- -e prod + - cd acceptance-tests && npm test -- -e dev - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[dev] From 90d1afbc09e8582f558740ca1b8e8434b88f9d40 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Wed, 28 Oct 2015 09:49:07 -0400 Subject: [PATCH 14/17] - separate prod and prod_build env deploys - run acceptance tests after deploy --- circle.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index 0fd2eaf7..12b762c5 100644 --- a/circle.yml +++ b/circle.yml @@ -9,18 +9,23 @@ dependencies: - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json deployment: + prod_build: + branch: production + commands: + - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install + - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install + - cd pelias-deploy && bundle exec rake deploy:api[prod_build] prod: branch: production commands: - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install - - cd acceptance-tests && npm test -- -e prod - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[prod] - - cd pelias-deploy && bundle exec rake deploy:api[prod_build] + - cd acceptance-tests && npm test -- -e prod dev: branch: master commands: - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install - - cd acceptance-tests && npm test -- -e dev - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[dev] + - cd acceptance-tests && npm test -- -e dev From fdcdea32581480e5b1f0e9228fd19d17937e3e42 Mon Sep 17 00:00:00 2001 From: Grant Heffernan Date: Wed, 28 Oct 2015 14:02:01 -0400 Subject: [PATCH 15/17] simplify circle deployment --- circle.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/circle.yml b/circle.yml index 12b762c5..9a9b5bc7 100644 --- a/circle.yml +++ b/circle.yml @@ -9,12 +9,6 @@ dependencies: - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json deployment: - prod_build: - branch: production - commands: - - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install - - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - - cd pelias-deploy && bundle exec rake deploy:api[prod_build] prod: branch: production commands: @@ -22,6 +16,7 @@ deployment: - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[prod] - cd acceptance-tests && npm test -- -e prod + - cd pelias-deploy && bundle exec rake deploy:api[prod_build] dev: branch: master commands: From ac01d72477be0f13132653e673a06648cbb38b17 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Mon, 2 Nov 2015 12:55:35 +0100 Subject: [PATCH 16/17] update address matching analyzers. related https://github.com/pelias/schema/pull/77 --- query/autocomplete_defaults.js | 6 +++--- query/reverse_defaults.js | 6 +++--- query/search_defaults.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index b61e84ac..f6a9638c 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -38,15 +38,15 @@ module.exports = extend( false, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', - 'address:housenumber:analyzer': 'standard', + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address.number', 'address:housenumber:boost': 2, - 'address:street:analyzer': 'standard', + 'address:street:analyzer': 'peliasStreet', 'address:street:field': 'address.street', 'address:street:boost': 5, - 'address:postcode:analyzer': 'standard', + 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', 'address:postcode:boost': 3, diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index b61e84ac..f6a9638c 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -38,15 +38,15 @@ module.exports = extend( false, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', - 'address:housenumber:analyzer': 'standard', + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address.number', 'address:housenumber:boost': 2, - 'address:street:analyzer': 'standard', + 'address:street:analyzer': 'peliasStreet', 'address:street:field': 'address.street', 'address:street:boost': 5, - 'address:postcode:analyzer': 'standard', + 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', 'address:postcode:boost': 3, diff --git a/query/search_defaults.js b/query/search_defaults.js index b61e84ac..f6a9638c 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -38,15 +38,15 @@ module.exports = extend( false, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', - 'address:housenumber:analyzer': 'standard', + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address.number', 'address:housenumber:boost': 2, - 'address:street:analyzer': 'standard', + 'address:street:analyzer': 'peliasStreet', 'address:street:field': 'address.street', 'address:street:boost': 5, - 'address:postcode:analyzer': 'standard', + 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', 'address:postcode:boost': 3, From c17a0dbc54220d5f8929fa81a30c45612c69e700 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Mon, 2 Nov 2015 16:33:21 +0100 Subject: [PATCH 17/17] increase postcode weights, add postcode to leftovers --- helper/adminFields.js | 5 +++-- query/autocomplete_defaults.js | 2 +- query/search_defaults.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/helper/adminFields.js b/helper/adminFields.js index 275e03c1..af6ac432 100644 --- a/helper/adminFields.js +++ b/helper/adminFields.js @@ -10,7 +10,8 @@ var ADMIN_FIELDS = [ 'admin2', 'local_admin', 'locality', - 'neighborhood' + 'neighborhood', + 'address.zip' ]; /** @@ -41,4 +42,4 @@ function getAvailableAdminFields(schema, expectedFields, logger) { return available; } -module.exports = getAvailableAdminFields; \ No newline at end of file +module.exports = getAvailableAdminFields; diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index f6a9638c..401ebf73 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -48,7 +48,7 @@ module.exports = extend( false, peliasQuery.defaults, { 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', - 'address:postcode:boost': 3, + 'address:postcode:boost': 20, 'admin:alpha3:analyzer': 'standard', 'admin:alpha3:field': 'alpha3', diff --git a/query/search_defaults.js b/query/search_defaults.js index f6a9638c..401ebf73 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -48,7 +48,7 @@ module.exports = extend( false, peliasQuery.defaults, { 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', - 'address:postcode:boost': 3, + 'address:postcode:boost': 20, 'admin:alpha3:analyzer': 'standard', 'admin:alpha3:field': 'alpha3',