diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..9a9b5bc7 --- /dev/null +++ b/circle.yml @@ -0,0 +1,26 @@ +machine: + ruby: + version: 2.1.2 + node: + version: 0.12.2 + +dependencies: + pre: + - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json + +deployment: + prod: + branch: production + commands: + - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install + - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install + - cd pelias-deploy && bundle exec rake deploy:api[prod] + - cd acceptance-tests && npm test -- -e prod + - cd pelias-deploy && bundle exec rake deploy:api[prod_build] + dev: + branch: master + commands: + - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install + - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install + - cd pelias-deploy && bundle exec rake deploy:api[dev] + - cd acceptance-tests && npm test -- -e dev diff --git a/helper/adminFields.js b/helper/adminFields.js index 275e03c1..af6ac432 100644 --- a/helper/adminFields.js +++ b/helper/adminFields.js @@ -10,7 +10,8 @@ var ADMIN_FIELDS = [ 'admin2', 'local_admin', 'locality', - 'neighborhood' + 'neighborhood', + 'address.zip' ]; /** @@ -41,4 +42,4 @@ function getAvailableAdminFields(schema, expectedFields, logger) { return available; } -module.exports = getAvailableAdminFields; \ No newline at end of file +module.exports = getAvailableAdminFields; diff --git a/helper/query_parser.js b/helper/text_parser.js similarity index 93% rename from helper/query_parser.js rename to helper/text_parser.js index cb5dc7a7..a561aef9 100644 --- a/helper/query_parser.js +++ b/helper/text_parser.js @@ -8,6 +8,10 @@ var logger = require('pelias-logger').get('api'); module.exports = {}; +/* + * For performance, and to prefer POI and admin records, express a preference + * to only search coarse layers on very short text inputs. + */ module.exports.get_layers = function get_layers(query) { if (query.length <= 3 ) { // no address parsing required diff --git a/helper/types.js b/helper/types.js index 610d1aba..d9dec59d 100644 --- a/helper/types.js +++ b/helper/types.js @@ -2,13 +2,15 @@ var type_mapping = require( '../helper/type_mapping' ); var _ = require('lodash'); /** - * Combine all types and determine the unique subset + * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched + * This method decides how to combine all the preferences. * * @param {Array} clean_types * @returns {Array} */ module.exports = function calculate_types(clean_types) { - if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_address_parser)) { + //Check that at least one preference of types is defined + if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) { throw new Error('clean_types should not be null or undefined'); } @@ -33,8 +35,8 @@ module.exports = function calculate_types(clean_types) { * Type restrictions requested by the address parser should only be used * if both the source and layers parameters are empty, so do this last */ - if (clean_types.from_address_parser) { - return clean_types.from_address_parser; + if (clean_types.from_text_parser) { + return clean_types.from_text_parser; } throw new Error('no types specified'); diff --git a/package.json b/package.json index da77a3df..be08ea46 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "start": "node index.js", "test": "npm run unit", - "unit": "node test/unit/run.js | tap-spec", + "unit": "node test/unit/run.js | tap-dot", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", @@ -64,7 +64,7 @@ "nsp": "^0.3.0", "precommit-hook": "^1.0.7", "proxyquire": "^1.4.0", - "tap-spec": "^0.2.0", + "tap-dot": "^1.0.0", "tape": "^2.13.4" } } diff --git a/query/autocomplete.js b/query/autocomplete.js index ad64413a..0e9b9d68 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -1,6 +1,6 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./autocomplete_defaults'), check = require('check-types'); //------------------------------ diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js new file mode 100644 index 00000000..401ebf73 --- /dev/null +++ b/query/autocomplete_defaults.js @@ -0,0 +1,95 @@ + +var peliasQuery = require('pelias-query'), + extend = require('extend'); + +module.exports = extend( false, peliasQuery.defaults, { + + 'size': 10, + 'track_scores': true, + + 'centroid:field': 'center_point', + + 'sort:distance:order': 'asc', + 'sort:distance:distance_type': 'plane', + + 'boundary:circle:radius': '50km', + 'boundary:circle:distance_type': 'plane', + 'boundary:circle:optimize_bbox': 'indexed', + 'boundary:circle:_cache': true, + + 'boundary:rect:type': 'indexed', + 'boundary:rect:_cache': true, + + 'ngram:analyzer': 'peliasOneEdgeGram', + 'ngram:field': 'name.default', + 'ngram:boost': 1, + + 'phrase:analyzer': 'peliasPhrase', + 'phrase:field': 'phrase.default', + 'phrase:boost': 1, + 'phrase:slop': 2, + + 'focus:function': 'linear', + 'focus:offset': '1km', + 'focus:scale': '50km', + 'focus:decay': 0.5, + 'focus:weight': 2, + + 'function_score:score_mode': 'avg', + 'function_score:boost_mode': 'replace', + + 'address:housenumber:analyzer': 'peliasHousenumber', + 'address:housenumber:field': 'address.number', + 'address:housenumber:boost': 2, + + 'address:street:analyzer': 'peliasStreet', + 'address:street:field': 'address.street', + 'address:street:boost': 5, + + 'address:postcode:analyzer': 'peliasZip', + 'address:postcode:field': 'address.zip', + 'address:postcode:boost': 20, + + 'admin:alpha3:analyzer': 'standard', + 'admin:alpha3:field': 'alpha3', + 'admin:alpha3:boost': 5, + + 'admin:admin0:analyzer': 'peliasAdmin', + 'admin:admin0:field': 'admin0', + 'admin:admin0:boost': 4, + + 'admin:admin1:analyzer': 'peliasAdmin', + 'admin:admin1:field': 'admin1', + 'admin:admin1:boost': 3, + + 'admin:admin1_abbr:analyzer': 'peliasAdmin', + 'admin:admin1_abbr:field': 'admin1_abbr', + 'admin:admin1_abbr:boost': 3, + + 'admin:admin2:analyzer': 'peliasAdmin', + 'admin:admin2:field': 'admin2', + 'admin:admin2:boost': 2, + + 'admin:local_admin:analyzer': 'peliasAdmin', + 'admin:local_admin:field': 'local_admin', + 'admin:local_admin:boost': 1, + + 'admin:locality:analyzer': 'peliasAdmin', + 'admin:locality:field': 'locality', + 'admin:locality:boost': 1, + + 'admin:neighborhood:analyzer': 'peliasAdmin', + 'admin:neighborhood:field': 'neighborhood', + 'admin:neighborhood:boost': 1, + + 'popularity:field': 'popularity', + 'popularity:modifier': 'log1p', + 'popularity:max_boost': 20, + 'popularity:weight': 1, + + 'population:field': 'population', + 'population:modifier': 'log1p', + 'population:max_boost': 20, + 'population:weight': 2 + +}); diff --git a/query/reverse.js b/query/reverse.js index fd84b427..b48597c1 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,5 +1,5 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./reverse_defaults'), check = require('check-types'); //------------------------------ diff --git a/query/defaults.js b/query/reverse_defaults.js similarity index 94% rename from query/defaults.js rename to query/reverse_defaults.js index b61e84ac..f6a9638c 100644 --- a/query/defaults.js +++ b/query/reverse_defaults.js @@ -38,15 +38,15 @@ module.exports = extend( false, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', - 'address:housenumber:analyzer': 'standard', + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address.number', 'address:housenumber:boost': 2, - 'address:street:analyzer': 'standard', + 'address:street:analyzer': 'peliasStreet', 'address:street:field': 'address.street', 'address:street:boost': 5, - 'address:postcode:analyzer': 'standard', + 'address:postcode:analyzer': 'peliasZip', 'address:postcode:field': 'address.zip', 'address:postcode:boost': 3, diff --git a/query/search.js b/query/search.js index a30aa2fe..da7773a9 100644 --- a/query/search.js +++ b/query/search.js @@ -1,5 +1,5 @@ var peliasQuery = require('pelias-query'), - defaults = require('./defaults'), + defaults = require('./search_defaults'), textParser = require('./text_parser'), check = require('check-types'), geolib = require('geolib'); diff --git a/query/search_defaults.js b/query/search_defaults.js new file mode 100644 index 00000000..401ebf73 --- /dev/null +++ b/query/search_defaults.js @@ -0,0 +1,95 @@ + +var peliasQuery = require('pelias-query'), + extend = require('extend'); + +module.exports = extend( false, peliasQuery.defaults, { + + 'size': 10, + 'track_scores': true, + + 'centroid:field': 'center_point', + + 'sort:distance:order': 'asc', + 'sort:distance:distance_type': 'plane', + + 'boundary:circle:radius': '50km', + 'boundary:circle:distance_type': 'plane', + 'boundary:circle:optimize_bbox': 'indexed', + 'boundary:circle:_cache': true, + + 'boundary:rect:type': 'indexed', + 'boundary:rect:_cache': true, + + 'ngram:analyzer': 'peliasOneEdgeGram', + 'ngram:field': 'name.default', + 'ngram:boost': 1, + + 'phrase:analyzer': 'peliasPhrase', + 'phrase:field': 'phrase.default', + 'phrase:boost': 1, + 'phrase:slop': 2, + + 'focus:function': 'linear', + 'focus:offset': '1km', + 'focus:scale': '50km', + 'focus:decay': 0.5, + 'focus:weight': 2, + + 'function_score:score_mode': 'avg', + 'function_score:boost_mode': 'replace', + + 'address:housenumber:analyzer': 'peliasHousenumber', + 'address:housenumber:field': 'address.number', + 'address:housenumber:boost': 2, + + 'address:street:analyzer': 'peliasStreet', + 'address:street:field': 'address.street', + 'address:street:boost': 5, + + 'address:postcode:analyzer': 'peliasZip', + 'address:postcode:field': 'address.zip', + 'address:postcode:boost': 20, + + 'admin:alpha3:analyzer': 'standard', + 'admin:alpha3:field': 'alpha3', + 'admin:alpha3:boost': 5, + + 'admin:admin0:analyzer': 'peliasAdmin', + 'admin:admin0:field': 'admin0', + 'admin:admin0:boost': 4, + + 'admin:admin1:analyzer': 'peliasAdmin', + 'admin:admin1:field': 'admin1', + 'admin:admin1:boost': 3, + + 'admin:admin1_abbr:analyzer': 'peliasAdmin', + 'admin:admin1_abbr:field': 'admin1_abbr', + 'admin:admin1_abbr:boost': 3, + + 'admin:admin2:analyzer': 'peliasAdmin', + 'admin:admin2:field': 'admin2', + 'admin:admin2:boost': 2, + + 'admin:local_admin:analyzer': 'peliasAdmin', + 'admin:local_admin:field': 'local_admin', + 'admin:local_admin:boost': 1, + + 'admin:locality:analyzer': 'peliasAdmin', + 'admin:locality:field': 'locality', + 'admin:locality:boost': 1, + + 'admin:neighborhood:analyzer': 'peliasAdmin', + 'admin:neighborhood:field': 'neighborhood', + 'admin:neighborhood:boost': 1, + + 'popularity:field': 'popularity', + 'popularity:modifier': 'log1p', + 'popularity:max_boost': 20, + 'popularity:weight': 1, + + 'population:field': 'population', + 'population:modifier': 'log1p', + 'population:max_boost': 20, + 'population:weight': 2 + +}); diff --git a/sanitiser/_geo_reverse.js b/sanitiser/_geo_reverse.js index c15af48b..52f2980f 100644 --- a/sanitiser/_geo_reverse.js +++ b/sanitiser/_geo_reverse.js @@ -1,7 +1,7 @@ var geo_common = require ('./_geo_common'); var _ = require('lodash'); -var defaults = require('../query/defaults'); +var defaults = require('../query/reverse_defaults'); var LAT_LON_IS_REQUIRED = true, CIRCLE_IS_REQUIRED = false; diff --git a/sanitiser/_text.js b/sanitiser/_text.js index d0cc0986..824c7a87 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -1,5 +1,5 @@ var check = require('check-types'), - query_parser = require('../helper/query_parser'); + text_parser = require('../helper/text_parser'); // validate texts, convert types and apply defaults function sanitize( raw, clean ){ @@ -19,14 +19,14 @@ function sanitize( raw, clean ){ clean.text = raw.text; // parse text with query parser - var parsed_text = query_parser.get_parsed_address(clean.text); + var parsed_text = text_parser.get_parsed_address(clean.text); if (check.assigned(parsed_text)) { clean.parsed_text = parsed_text; } // try to set layers from query parser results clean.types = clean.layers || {}; - clean.types.from_address_parsing = query_parser.get_layers(clean.text); + clean.types.from_text_parser = text_parser.get_layers(clean.text); } return messages; diff --git a/test/unit/helper/query_parser.js b/test/unit/helper/text_parser.js similarity index 98% rename from test/unit/helper/query_parser.js rename to test/unit/helper/text_parser.js index e7771ca4..a4ce64e3 100644 --- a/test/unit/helper/query_parser.js +++ b/test/unit/helper/text_parser.js @@ -1,4 +1,4 @@ -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); var type_mapping = require('../../../helper/type_mapping'); var layers_map = type_mapping.layer_with_aliases_to_type; diff --git a/test/unit/helper/types.js b/test/unit/helper/types.js index 4638ff12..92848d46 100644 --- a/test/unit/helper/types.js +++ b/test/unit/helper/types.js @@ -33,7 +33,7 @@ module.exports.tests.no_cleaned_types = function(test, common) { module.exports.tests.address_parser = function(test, common) { test('address parser specifies only admin layers', function(t) { var cleaned_types = { - from_address_parser: ['admin0'] // simplified return value from address parser + from_text_parser: ['admin0'] // simplified return value from address parser }; var actual = types(cleaned_types); var expected = ['admin0']; // simplified expected value for all admin layers @@ -58,7 +58,7 @@ module.exports.tests.layers_parameter_and_address_parser = function(test, common test('layers parameter and address parser present', function(t) { var cleaned_types = { from_layers: ['geoname'], - from_address_parser: ['admin0'] // simplified return value from address parse + from_text_parser: ['admin0'] // simplified return value from address parse }; var actual = types(cleaned_types); var expected = ['geoname']; diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js index ecdf9553..c3f620c0 100644 --- a/test/unit/query/autocomplete.js +++ b/test/unit/query/autocomplete.js @@ -1,6 +1,6 @@ var generate = require('../../../query/autocomplete'); -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); module.exports.tests = {}; diff --git a/test/unit/query/autocomplete_defaults.js b/test/unit/query/autocomplete_defaults.js new file mode 100644 index 00000000..7ccb17a0 --- /dev/null +++ b/test/unit/query/autocomplete_defaults.js @@ -0,0 +1,22 @@ + +var defaults = require('../../../query/autocomplete_defaults'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof defaults, 'object', 'defaults defined'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('autocomplete defaults ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/reverse_defaults.js b/test/unit/query/reverse_defaults.js new file mode 100644 index 00000000..48a9805f --- /dev/null +++ b/test/unit/query/reverse_defaults.js @@ -0,0 +1,22 @@ + +var defaults = require('../../../query/reverse_defaults'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof defaults, 'object', 'defaults defined'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('reverse defaults ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/search.js b/test/unit/query/search.js index 3a11e054..15527222 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -1,5 +1,5 @@ var generate = require('../../../query/search'); -var parser = require('../../../helper/query_parser'); +var parser = require('../../../helper/text_parser'); module.exports.tests = {}; diff --git a/test/unit/query/defaults.js b/test/unit/query/search_defaults.js similarity index 77% rename from test/unit/query/defaults.js rename to test/unit/query/search_defaults.js index d6fcc720..54b116b7 100644 --- a/test/unit/query/defaults.js +++ b/test/unit/query/search_defaults.js @@ -1,5 +1,5 @@ -var defaults = require('../../../query/defaults'); +var defaults = require('../../../query/search_defaults'); module.exports.tests = {}; @@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) { module.exports.all = function (tape, common) { function test(name, testFunction) { - return tape('query defaults ' + name, testFunction); + return tape('search defaults ' + name, testFunction); } for( var testCase in module.exports.tests ){ diff --git a/test/unit/run.js b/test/unit/run.js index 7e0fe272..76a3a437 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -9,14 +9,16 @@ var tests = [ require('./helper/geojsonify'), require('./helper/labelGenerator'), require('./helper/labelSchema'), - require('./helper/query_parser'), + require('./helper/text_parser'), require('./helper/type_mapping'), require('./helper/types'), require('./middleware/confidenceScore'), require('./middleware/confidenceScoreReverse'), require('./middleware/distance'), require('./query/autocomplete'), - require('./query/defaults'), + require('./query/autocomplete_defaults'), + require('./query/search_defaults'), + require('./query/reverse_defaults'), require('./query/reverse'), require('./query/search'), require('./sanitiser/_boundary_country'), @@ -29,6 +31,7 @@ var tests = [ require('./sanitiser/_single_scalar_parameters'), require('./sanitiser/_size'), require('./sanitiser/_sources'), + require('./sanitiser/_text'), require('./sanitiser/autocomplete'), require('./sanitiser/place'), require('./sanitiser/reverse'), diff --git a/test/unit/sanitiser/_geo_reverse.js b/test/unit/sanitiser/_geo_reverse.js index e03e7ddd..e7c62e07 100644 --- a/test/unit/sanitiser/_geo_reverse.js +++ b/test/unit/sanitiser/_geo_reverse.js @@ -1,5 +1,5 @@ var sanitize = require('../../../sanitiser/_geo_reverse'); -var defaults = require('../../../query/defaults'); +var defaults = require('../../../query/reverse_defaults'); module.exports.tests = {}; diff --git a/test/unit/sanitiser/_text.js b/test/unit/sanitiser/_text.js new file mode 100644 index 00000000..8bc0710f --- /dev/null +++ b/test/unit/sanitiser/_text.js @@ -0,0 +1,32 @@ +var sanitiser = require('../../../sanitiser/_text'); +var type_mapping = require('../../../helper/type_mapping'); + +module.exports.tests = {}; + +module.exports.tests.text_parser = function(test, common) { + test('short input text has admin layers set ', function(t) { + var raw = { + text: 'emp' //start of empire state building + }; + var clean = { + }; + + var messages = sanitiser(raw, clean); + + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + t.equal(clean.types.from_text_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred'); + + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANITISER _text: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/sanitiser/reverse.js b/test/unit/sanitiser/reverse.js index b92896e7..4e727b5b 100644 --- a/test/unit/sanitiser/reverse.js +++ b/test/unit/sanitiser/reverse.js @@ -4,7 +4,7 @@ var reverse = require('../../../sanitiser/reverse'), sanitize = reverse.sanitize, middleware = reverse.middleware, - defaults = require('../../../query/defaults'), + defaults = require('../../../query/reverse_defaults'), defaultError = 'missing param \'lat\'', defaultClean = { 'point.lat': 0, 'point.lon': 0, diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index 4f4e2880..b00160dd 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -1,6 +1,6 @@ var extend = require('extend'), search = require('../../../sanitiser/search'), - parser = require('../../../helper/query_parser'), + parser = require('../../../helper/text_parser'), sanitize = search.sanitize, middleware = search.middleware, defaultError = 'invalid param \'text\': text length, must be >0';