diff --git a/circle.yml b/circle.yml index 3c429ae3..37a6e3d7 100644 --- a/circle.yml +++ b/circle.yml @@ -4,20 +4,14 @@ machine: node: version: 0.12.2 -dependencies: - pre: - - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json - deployment: dev: branch: master commands: - - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[dev] - - cd acceptance-tests && npm test -- -e dev prod_build: - branch: production + branch: staging commands: - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install - cd pelias-deploy && bundle exec rake deploy:api[prod_build] diff --git a/controller/search.js b/controller/search.js index 93767713..309e7647 100644 --- a/controller/search.js +++ b/controller/search.js @@ -1,4 +1,5 @@ var service = { search: require('../service/search') }; +var logger = require('pelias-logger').get('api:controller:search'); function setup( backend, query ){ @@ -14,6 +15,9 @@ function setup( backend, query ){ return next(); } + // log clean parameters for stats + logger.info('[req]', 'endpoint=' + req.path, req.clean); + // backend command var cmd = { index: 'pelias', diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index 68b85c0d..b100b973 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -100,8 +100,8 @@ function checkForDealBreakers(req, hit) { return true; } - if (check.assigned(req.clean.parsed_text.postalcode) && req.clean.parsed_text.postalcode !== hit.zip) { - logger.debug('[confidence][deal-breaker]: postalcode !== zip'); + if (check.assigned(req.clean.parsed_text.postalcode) && req.clean.parsed_text.postalcode !== hit.address.zip) { + logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')'); return true; } } @@ -208,7 +208,7 @@ function propMatch(textProp, hitProp, expectEnriched) { * @param {object} [hit.address] * @param {string|number} [hit.address.number] * @param {string} [hit.address.street] - * @param {string|number} [hit.zip] + * @param {string|number} [hit.address.zip] * @param {string} [hit.admin1_abbr] * @param {string} [hit.alpha3] * @returns {number} diff --git a/middleware/dedupe.js b/middleware/dedupe.js new file mode 100644 index 00000000..b02ca2a3 --- /dev/null +++ b/middleware/dedupe.js @@ -0,0 +1,99 @@ +var logger = require('pelias-logger').get('api'); +var _ = require('lodash'); + +function setup() { + return dedupeResults; +} + +function dedupeResults(req, res, next) { + // do nothing if no result data set + if (_.isUndefined(req.clean) || _.isUndefined(res) || _.isUndefined(res.data)) { + return next(); + } + + // loop through data items and only copy unique items to uniqueResults + var uniqueResults = []; + + _.some(res.data, function (hit) { + if (uniqueResults.length === 0 || _.every(uniqueResults, isDifferent.bind(null, hit)) ) { + uniqueResults.push(hit); + } + else { + logger.info('[dupe]', { query: req.clean.text, hit: hit.name.default }); + } + + // stop looping when requested size has been reached in uniqueResults + return req.clean.size <= uniqueResults.length; + }); + + res.data = uniqueResults; + + next(); +} + +/** + * @param {object} item1 + * @param {object} item2 + * @returns {boolean} + * @throws {Error} + */ +function isDifferent(item1, item2) { + try { + propMatch(item1, item2, 'admin1_abbr'); + propMatch(item1, item2, 'alpha3'); + + if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) { + propMatch(item1.name, item2.name, 'default'); + } + else { + propMatch(item1, item2, 'name'); + } + + if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) { + propMatch(item1.address, item2.address, 'number'); + propMatch(item1.address, item2.address, 'street'); + propMatch(item1.address, item2.address, 'zip'); + } + else if (item1.address !== item2.address) { + throw new Error('different'); + } + } + catch (err) { + if (err.message === 'different') { + return true; + } + throw err; + } + + return false; +} + +/** + * Throw exception if properties are different + * + * @param {object} item1 + * @param {object} item2 + * @param {string} prop + * @throws {Error} + */ +function propMatch(item1, item2, prop) { + if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) { + throw new Error('different'); + } +} + +/** + * Remove punctuation and lowercase + * + * @param {string} str + * @returns {string} + */ +function normalizeString(str) { + if (!str) { + return ''; + } + return _.words(str.toLowerCase()).join(' '); +} + + +module.exports = setup; \ No newline at end of file diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js new file mode 100644 index 00000000..88334d3b --- /dev/null +++ b/middleware/sizeCalculator.js @@ -0,0 +1,35 @@ +var _ = require('lodash'); + +var SIZE_PADDING = 2; + +/** + * Utility for calculating query result size + * incorporating padding for dedupe process + */ +function setup() { + return function setQuerySize(req, res, next) { + if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { + return next(); + } + + req.clean.querySize = calculateSize(req.clean.size); + next(); + }; +} + +/** + * Add padding or set to 1 + * + * @param {number} cleanSize + * @returns {number} + */ +function calculateSize(cleanSize) { + switch (cleanSize || 1) { + case 1: + return 1; + default: + return cleanSize * SIZE_PADDING; + } +} + +module.exports = setup; \ No newline at end of file diff --git a/package.json b/package.json index be08ea46..eb9a4b10 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "elasticsearch": ">=1.2.1" }, "dependencies": { - "addressit": "1.3.0", + "addressit": "git://github.com/dianashk/addressit.git#temp", "async": "^0.9.0", "check-types": "^3.3.1", "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", diff --git a/query/autocomplete.js b/query/autocomplete.js index 0e9b9d68..ca4acdcb 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -30,9 +30,6 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); - // always 10 (not user definable due to caching) - vs.var( 'size', 10 ); - // focus point if( check.number(clean['focus.point.lat']) && check.number(clean['focus.point.lon']) ){ diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 401ebf73..28cf6e7a 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 20, 'track_scores': true, 'centroid:field': 'center_point', diff --git a/query/reverse.js b/query/reverse.js index b48597c1..856b9540 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -22,15 +22,9 @@ function generateQuery( clean ){ var vs = new peliasQuery.Vars( defaults ); - // set defaults - vs.set({ - 'size': 1, - 'boundary:circle:radius': '500km' - }); - // set size - if( clean.size ){ - vs.var( 'size', clean.size ); + if( clean.querySize ){ + vs.var( 'size', clean.querySize); } // focus point to score by distance diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index f6a9638c..01e74773 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 1, 'track_scores': true, 'centroid:field': 'center_point', @@ -12,7 +12,7 @@ module.exports = extend( false, peliasQuery.defaults, { 'sort:distance:order': 'asc', 'sort:distance:distance_type': 'plane', - 'boundary:circle:radius': '50km', + 'boundary:circle:radius': '500km', 'boundary:circle:distance_type': 'plane', 'boundary:circle:optimize_bbox': 'indexed', 'boundary:circle:_cache': true, diff --git a/query/search.js b/query/search.js index da7773a9..91999b1e 100644 --- a/query/search.js +++ b/query/search.js @@ -52,8 +52,8 @@ function generateQuery( clean ){ vs.var( 'input:name', clean.text ); // size - if( clean.size ){ - vs.var( 'size', clean.size ); + if( clean.querySize ) { + vs.var( 'size', clean.querySize ); } // focus point diff --git a/query/search_defaults.js b/query/search_defaults.js index 401ebf73..28cf6e7a 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 20, 'track_scores': true, 'centroid:field': 'center_point', diff --git a/routes/v1.js b/routes/v1.js index 4d112458..49c9ea4c 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -12,7 +12,8 @@ var sanitisers = { /** ----------------------- middleware ------------------------ **/ var middleware = { - types: require('../middleware/_types') + types: require('../middleware/_types'), + calcSize: require('../middleware/sizeCalculator') }; /** ----------------------- controllers ----------------------- **/ @@ -30,6 +31,7 @@ var postProc = { distances: require('../middleware/distance'), confidenceScores: require('../middleware/confidenceScore'), confidenceScoresReverse: require('../middleware/confidenceScoreReverse'), + dedupe: require('../middleware/dedupe'), localNamingConventions: require('../middleware/localNamingConventions'), renamePlacenames: require('../middleware/renamePlacenames'), geocodeJSON: require('../middleware/geocodeJSON'), @@ -58,9 +60,11 @@ function addRoutes(app, peliasConfig) { search: createRouter([ sanitisers.search.middleware, middleware.types, + middleware.calcSize(), controllers.search(), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), @@ -72,6 +76,7 @@ function addRoutes(app, peliasConfig) { controllers.search(null, require('../query/autocomplete')), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), @@ -80,11 +85,13 @@ function addRoutes(app, peliasConfig) { reverse: createRouter([ sanitisers.reverse.middleware, middleware.types, + middleware.calcSize(), controllers.search(undefined, reverseQuery), postProc.distances('point.'), // reverse confidence scoring depends on distance from origin // so it must be calculated first postProc.confidenceScoresReverse(), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), diff --git a/test/ciao/search/address_parsing.coffee b/test/ciao/search/address_parsing.coffee index 83d3e1bc..e39f484e 100644 --- a/test/ciao/search/address_parsing.coffee +++ b/test/ciao/search/address_parsing.coffee @@ -34,7 +34,7 @@ json.geocoding.query['size'].should.eql 10 #? address parsing json.geocoding.query.parsed_text['name'].should.eql '30 w 26th st' -json.geocoding.query.parsed_text['number'].should.eql 30 +json.geocoding.query.parsed_text['number'].should.eql '30' json.geocoding.query.parsed_text['street'].should.eql 'w 26th st' json.geocoding.query.parsed_text['state'].should.eql 'NY' json.geocoding.query.parsed_text['regions'].should.eql [] diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 8d0e9914..0ef70786 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -115,6 +116,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 8300b026..59b72061 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -115,6 +116,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 53ece28e..e20170c5 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -87,6 +88,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js new file mode 100644 index 00000000..6ed761c4 --- /dev/null +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -0,0 +1,273 @@ +module.exports = [ + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.039265 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.039265 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter, High-School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.23246, + 'lat': 39.99288 + }, + 'address': {}, + 'local_admin': 'West Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'Lampeter-Strasburg High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Lampeter', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Wheatland Mills', + 'category': [ + 'education' + ], + '_id': '4559068', + '_type': 'geoname', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.20746, + 'lat': 40.03927 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '5187980', + '_type': 'geoname', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.232457, + 'lat': 39.992877 + }, + 'address': {}, + 'local_admin': 'West Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'Lampeter-Strasburg High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Lampeter', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Wheatland Mills', + 'category': [ + 'education' + ], + '_id': '357294404', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.038987 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357283977', + '_type': 'osmnode', + '_score': 1.1036991, + 'confidence': 0.664 + }, + { + 'center_point': { + 'lon': -76.20746, + 'lat': 40.03899 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '5187966', + '_type': 'geoname', + '_score': 1.1036991, + 'confidence': 0.664 + }, + { + 'center_point': { + 'lon': -94.167445, + 'lat': 38.762788 + }, + 'address': {}, + 'local_admin': 'Polk', + 'admin1_abbr': 'MO', + 'name': { + 'default': 'Strasburg School' + }, + 'admin1': 'Missouri', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Cass County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '358058986', + '_type': 'osmnode', + '_score': 1.0492544, + 'confidence': 0.658 + }, + { + 'center_point': { + 'lon': -78.36317, + 'lat': 38.98445 + }, + 'address': {}, + 'admin1_abbr': 'VA', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'Virginia', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Shenandoah County', + 'admin0': 'United States', + 'neighborhood': 'Strasburg Junction', + 'category': [ + 'education' + ], + '_id': '4787978', + '_type': 'geoname', + '_score': 0.9724125, + 'confidence': 0.649 + }, + { + 'center_point': { + 'lon': -100.16516, + 'lat': 46.13427 + }, + 'address': {}, + 'local_admin': 'Strasburg', + 'admin1_abbr': 'ND', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'North Dakota', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Emmons County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '9683163', + '_type': 'geoname', + '_score': 0.9724125, + 'confidence': 0.649 + }, + { + 'center_point': { + 'lon': -81.532392, + 'lat': 40.597578 + }, + 'address': {}, + 'local_admin': 'Franklin', + 'admin1_abbr': 'OH', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'Ohio', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Tuscarawas County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '356646971', + '_type': 'osmway', + '_score': 0.9724125, + 'confidence': 0.649 + } +]; \ No newline at end of file diff --git a/test/unit/fixture/reverse_boundary_circle.js b/test/unit/fixture/reverse_boundary_circle.js index b7cc058f..897f019f 100644 --- a/test/unit/fixture/reverse_boundary_circle.js +++ b/test/unit/fixture/reverse_boundary_circle.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -10,7 +11,7 @@ module.exports = { 'must': [ { 'geo_distance': { - 'distance': '500km', + 'distance': vs.distance, 'distance_type': 'plane', 'optimize_bbox': 'indexed', '_cache': true, @@ -38,6 +39,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; \ No newline at end of file diff --git a/test/unit/fixture/reverse_null_island.js b/test/unit/fixture/reverse_null_island.js index f29ab293..5f020ad2 100644 --- a/test/unit/fixture/reverse_null_island.js +++ b/test/unit/fixture/reverse_null_island.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -40,6 +41,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/reverse_standard.js b/test/unit/fixture/reverse_standard.js index 08a005e6..d0b0b734 100644 --- a/test/unit/fixture/reverse_standard.js +++ b/test/unit/fixture/reverse_standard.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -40,6 +41,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/reverse_with_boundary_country.js b/test/unit/fixture/reverse_with_boundary_country.js index f29569b2..8d739eaa 100644 --- a/test/unit/fixture/reverse_with_boundary_country.js +++ b/test/unit/fixture/reverse_with_boundary_country.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -49,6 +50,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 93572705..316903e7 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -1,6 +1,4 @@ - -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -88,89 +86,89 @@ module.exports = { },{ 'match': { 'address.number': { - 'query': 123, - 'boost': vs.var('address:housenumber:boost').get(), - 'analyzer': vs.var('address:housenumber:analyzer').get() + 'query': '123', + 'boost': vs['address:housenumber:boost'], + 'analyzer': vs['address:housenumber:analyzer'] } } }, { 'match': { 'address.street': { 'query': 'main st', - 'boost': vs.var('address:street:boost').get(), - 'analyzer': vs.var('address:street:analyzer').get() + 'boost': vs['address:street:boost'], + 'analyzer': vs['address:street:analyzer'] } } }, { 'match': { 'address.zip': { - 'query': 10010, - 'boost': vs.var('address:postcode:boost').get(), - 'analyzer': vs.var('address:postcode:analyzer').get() + 'query': '10010', + 'boost': vs['address:postcode:boost'], + 'analyzer': vs['address:postcode:analyzer'] } } }, { 'match': { 'alpha3': { 'query': 'USA', - 'boost': vs.var('admin:alpha3:boost').get(), - 'analyzer': vs.var('admin:alpha3:analyzer').get() + 'boost': vs['admin:alpha3:boost'], + 'analyzer': vs['admin:alpha3:analyzer'] } } }, { 'match': { 'admin0': { 'query': 'new york', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'new york', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'NY', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'new york', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'new york', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'new york', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'new york', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index 6e0fbd5b..745598da 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -1,6 +1,5 @@ -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -88,56 +87,56 @@ module.exports = { 'match': { 'admin0': { 'query': 'new york', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'new york', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'new york', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'new york', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'new york', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'new york', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'new york', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index cc04943b..51bb57cc 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -1,6 +1,5 @@ -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -87,73 +86,73 @@ module.exports = { },{ 'match': { 'address.number': { - 'query': 1, - 'boost': vs.var('address:housenumber:boost').get(), - 'analyzer': vs.var('address:housenumber:analyzer').get() + 'query': '1', + 'boost': vs['address:housenumber:boost'], + 'analyzer': vs['address:housenumber:analyzer'] } } }, { 'match': { 'address.street': { 'query': 'water st', - 'boost': vs.var('address:street:boost').get(), - 'analyzer': vs.var('address:street:analyzer').get() + 'boost': vs['address:street:boost'], + 'analyzer': vs['address:street:analyzer'] } } }, { 'match': { 'admin0': { 'query': 'manhattan', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'manhattan', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'NY', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'manhattan', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'manhattan', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'manhattan', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'manhattan', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js new file mode 100644 index 00000000..41d854d4 --- /dev/null +++ b/test/unit/helper/sizeCalculator.js @@ -0,0 +1,67 @@ + +var calcSize = require('../../../middleware/sizeCalculator.js')(); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof calcSize, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.tests.valid = function(test, common) { + var req = { clean: {} }; + function setup(val) { + if (isNaN(val)) { + delete req.clean.size; + } + else { + req.clean.size = val; + } + delete req.clean.querySize; + } + + test('size=0', function (t) { + setup(0); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 1); + t.end(); + }); + }); + + test('size=1', function (t) { + setup(1); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 1); + t.end(); + }); + }); + + test('size=10', function (t) { + setup(10); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 20); + t.end(); + }); + }); + + test('no size', function (t) { + setup(); + calcSize(req, {}, function () { + t.equal(req.clean.hasOwnProperty('querySize'), false); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('sizeCalculator: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/text_parser.js b/test/unit/helper/text_parser.js index a4ce64e3..3accf7b9 100644 --- a/test/unit/helper/text_parser.js +++ b/test/unit/helper/text_parser.js @@ -85,7 +85,7 @@ module.exports.tests.parse_address = function(test, common) { var address = parser.get_parsed_address(query_string); t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.number, 123, 'parsed house number'); + t.equal(address.number, '123', 'parsed house number'); t.equal(address.street, 'main st', 'parsed street'); t.deepEqual(address.regions, ['new york'], 'parsed city'); t.equal(address.state , 'NY', 'parsed state'); @@ -96,11 +96,23 @@ module.exports.tests.parse_address = function(test, common) { var address = parser.get_parsed_address(query_string); t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.number, 123, 'parsed house number'); + t.equal(address.number, '123', 'parsed house number'); t.equal(address.street, 'main st', 'parsed street'); t.deepEqual(address.regions, ['new york'], 'parsed city'); t.equal(address.state , 'NY', 'parsed state'); - t.equal(address.postalcode, 10010, 'parsed zip'); + t.equal(address.postalcode, '10010', 'parsed zip is a string'); + t.end(); + }); + test('valid address with leading 0s in zipcode', function(t) { + var query_string = '339 W Main St, Cheshire, 06410'; + var address = parser.get_parsed_address(query_string); + + console.log(address); + + t.equal(typeof address, 'object', 'valid object for the address'); + t.equal(address.street, 'W Main St', 'parsed street'); + t.deepEqual(address.regions, ['Cheshire'], 'parsed city'); + t.equal(address.postalcode, '06410', 'parsed zip'); t.end(); }); }; diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js new file mode 100644 index 00000000..31d4bf6f --- /dev/null +++ b/test/unit/middleware/dedupe.js @@ -0,0 +1,52 @@ +var data = require('../fixture/dedupe_elasticsearch_results'); +var dedupe = require('../../../middleware/dedupe')(); + +module.exports.tests = {}; + +module.exports.tests.dedupe = function(test, common) { + test('filter out duplicates', function(t) { + var req = { + clean: { + text: 'lampeter strasburg high school', + size: 100 + } + }; + var res = { + data: data + }; + + var expectedCount = 7; + dedupe(req, res, function () { + t.equal(res.data.length, expectedCount, 'results have fewer items than before'); + t.end(); + }); + }); + + test('truncate results based on specified size', function(t) { + var req = { + clean: { + text: 'lampeter strasburg high school', + size: 3 + } + }; + var res = { + data: data + }; + + dedupe(req, res, function () { + t.equal(res.data.length, req.clean.size, 'results have fewer items than before'); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('[middleware] dedupe: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js index 7a2f9d07..7e6c2ba1 100644 --- a/test/unit/query/reverse.js +++ b/test/unit/query/reverse.js @@ -79,14 +79,15 @@ module.exports.tests.query = function(test, common) { test('size fuzz test', function(t) { // test different sizes - var sizes = [1,2,10,undefined,null]; - sizes.forEach( function( size ){ + var sizes = [1,4,20,undefined,null]; + var expected = [1,4,20,1,1]; + sizes.forEach( function( size, index ){ var query = generate({ - 'point.lat': 29.49136, 'point.lon': -82.50622, size: size + 'point.lat': 29.49136, 'point.lon': -82.50622, querySize: size }); var compiled = JSON.parse( JSON.stringify( query ) ); - t.equal( compiled.size, size ? size : 1, 'valid reverse query for size: '+ size); + t.equal( compiled.size, expected[index], 'valid reverse query for size: '+ size); }); t.end(); }); diff --git a/test/unit/query/search.js b/test/unit/query/search.js index 15527222..7fa81979 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.query = function(test, common) { test('valid search + focus + bbox', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, 'boundary.rect.min_lat': 47.47, 'boundary.rect.max_lon': -61.84, @@ -31,7 +31,7 @@ module.exports.tests.query = function(test, common) { test('valid search + bbox', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'boundary.rect.min_lat': 47.47, 'boundary.rect.max_lon': -61.84, 'boundary.rect.max_lat': 11.51, @@ -48,7 +48,7 @@ module.exports.tests.query = function(test, common) { test('valid lingustic-only search', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, layers: ['test'] }); @@ -61,7 +61,7 @@ module.exports.tests.query = function(test, common) { test('search search + focus', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, layers: ['test'] }); @@ -75,7 +75,7 @@ module.exports.tests.query = function(test, common) { test('search search + viewport', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.viewport.min_lat': 28.49136, 'focus.viewport.max_lat': 30.49136, 'focus.viewport.min_lon': -87.50622, @@ -92,7 +92,7 @@ module.exports.tests.query = function(test, common) { test('search with viewport diagonal < 1km should set scale to 1km', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.viewport.min_lat': 28.49135, 'focus.viewport.max_lat': 28.49137, 'focus.viewport.min_lon': -87.50622, @@ -109,7 +109,7 @@ module.exports.tests.query = function(test, common) { test('search search + focus on null island', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 0, 'focus.point.lon': 0, layers: ['test'] }); @@ -126,7 +126,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(address), }); @@ -142,7 +142,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); @@ -158,7 +158,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); @@ -171,7 +171,7 @@ module.exports.tests.query = function(test, common) { test('valid boundary.country search', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, layers: ['test'], 'boundary.country': 'ABC' }); diff --git a/test/unit/run.js b/test/unit/run.js index 191dcf9c..0510cda6 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -12,10 +12,12 @@ var tests = [ require('./helper/text_parser'), require('./helper/type_mapping'), require('./helper/types'), + require('./helper/sizeCalculator'), require('./middleware/confidenceScore'), require('./middleware/confidenceScoreReverse'), require('./middleware/distance'), require('./middleware/localNamingConventions'), + require('./middleware/dedupe'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'),