From 54187dde679ccfd02c249187f9c0b25a4f89d043 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Tue, 1 Dec 2015 22:52:49 -0500 Subject: [PATCH 1/3] Add dedupe middleware Dedupe middleware removes __exact__ dupes and truncates the results to the specified size. --- controller/search.js | 4 + helper/sizeCalculator.js | 21 ++ middleware/dedupe.js | 82 ++++++ query/autocomplete.js | 5 +- query/reverse.js | 5 +- query/search.js | 8 +- routes/v1.js | 4 + .../fixture/autocomplete_linguistic_focus.js | 2 +- ...tocomplete_linguistic_focus_null_island.js | 2 +- .../fixture/autocomplete_linguistic_only.js | 2 +- .../fixture/dedupe_elasticsearch_results.js | 248 ++++++++++++++++++ test/unit/fixture/search_boundary_country.js | 2 +- test/unit/fixture/search_full_address.js | 2 +- test/unit/fixture/search_linguistic_bbox.js | 2 +- test/unit/fixture/search_linguistic_focus.js | 2 +- .../fixture/search_linguistic_focus_bbox.js | 2 +- .../search_linguistic_focus_null_island.js | 2 +- test/unit/fixture/search_linguistic_only.js | 2 +- .../fixture/search_linguistic_viewport.js | 2 +- ...search_linguistic_viewport_min_diagonal.js | 2 +- test/unit/fixture/search_partial_address.js | 2 +- test/unit/fixture/search_regions_address.js | 2 +- test/unit/helper/sizeCalculator.js | 39 +++ test/unit/middleware/dedupe.js | 52 ++++ test/unit/query/reverse.js | 5 +- test/unit/run.js | 2 + 26 files changed, 479 insertions(+), 24 deletions(-) create mode 100644 helper/sizeCalculator.js create mode 100644 middleware/dedupe.js create mode 100644 test/unit/fixture/dedupe_elasticsearch_results.js create mode 100644 test/unit/helper/sizeCalculator.js create mode 100644 test/unit/middleware/dedupe.js diff --git a/controller/search.js b/controller/search.js index 93767713..38bfb481 100644 --- a/controller/search.js +++ b/controller/search.js @@ -1,4 +1,5 @@ var service = { search: require('../service/search') }; +var logger = require('pelias-logger').get('api:controller:search'); function setup( backend, query ){ @@ -14,6 +15,9 @@ function setup( backend, query ){ return next(); } + // log clean parameters for stats + logger.info(req.clean); + // backend command var cmd = { index: 'pelias', diff --git a/helper/sizeCalculator.js b/helper/sizeCalculator.js new file mode 100644 index 00000000..b51bed76 --- /dev/null +++ b/helper/sizeCalculator.js @@ -0,0 +1,21 @@ +/** + * Utility for calculating query result size + * incorporating padding for dedupe process + */ + +var SIZE_PADDING = 2; + +/** + * Add padding or set to 1 + * + * @param {number} cleanSize + * @returns {number} + */ +module.exports = function calculateSize(cleanSize) { + switch (cleanSize || 1) { + case 1: + return 1; + default: + return cleanSize * SIZE_PADDING; + } +}; \ No newline at end of file diff --git a/middleware/dedupe.js b/middleware/dedupe.js new file mode 100644 index 00000000..c6e95c1c --- /dev/null +++ b/middleware/dedupe.js @@ -0,0 +1,82 @@ +var util = require('util'); +var logger = require('pelias-logger').get('api:middle:dedupe'); +var _ = require('lodash'); + +function setup() { + return dedupeResults; +} + +function dedupeResults(req, res, next) { + // do nothing if no result data set + if (_.isUndefined(req.clean) || _.isUndefined(res) || _.isUndefined(res.data)) { + return next(); + } + + // loop through data items and only copy unique items to uniqueResults + var uniqueResults = []; + + _.some(res.data, function (hit) { + if (uniqueResults.length === 0 || _.every(uniqueResults, isDifferent.bind(null, hit)) ) { + uniqueResults.push(hit); + } + else { + logger.info('[dupe]', { query: req.clean.text, hit: hit.name.default }); + } + + // stop looping when requested size has been reached in uniqueResults + return req.clean.size <= uniqueResults.length; + }); + + res.data = uniqueResults; + + next(); +} + +/** + * @param {object} item1 + * @param {object} item2 + * @returns {boolean} + */ +function isDifferent(item1, item2) { + try { + propMatch(item1, item2, 'admin1_abbr'); + propMatch(item1, item2, 'alpha3'); + + if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) { + propMatch(item1.name, item2.name, 'default'); + } + else if (item1.name !== item2.name) { + throw 'different'; + } + + if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) { + propMatch(item1.address, item2.address, 'number'); + propMatch(item1.address, item2.address, 'street'); + propMatch(item1.address, item2.address, 'zip'); + } + else if (item1.address !== item2.address) { + throw 'different'; + } + } + catch (err) { + return true; + } + + return false; +} + +/** + * Throw exception if properties are different + * + * @param item1 + * @param item2 + * @param prop + */ +function propMatch(item1, item2, prop) { + if (item1[prop] !== item2[prop]) { + throw 'different'; + } +} + + +module.exports = setup; \ No newline at end of file diff --git a/query/autocomplete.js b/query/autocomplete.js index 0e9b9d68..d9f3f3be 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -1,7 +1,8 @@ var peliasQuery = require('pelias-query'), defaults = require('./autocomplete_defaults'), - check = require('check-types'); + check = require('check-types'), + calcSize = require('../helper/sizeCalculator'); //------------------------------ // autocomplete query @@ -31,7 +32,7 @@ function generateQuery( clean ){ vs.var( 'input:name', clean.text ); // always 10 (not user definable due to caching) - vs.var( 'size', 10 ); + vs.var( 'size', calcSize(10)); // focus point if( check.number(clean['focus.point.lat']) && diff --git a/query/reverse.js b/query/reverse.js index b48597c1..5f196cbc 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,6 +1,7 @@ var peliasQuery = require('pelias-query'), defaults = require('./reverse_defaults'), - check = require('check-types'); + check = require('check-types'), + calcSize = require('../helper/sizeCalculator'); //------------------------------ // reverse geocode query @@ -30,7 +31,7 @@ function generateQuery( clean ){ // set size if( clean.size ){ - vs.var( 'size', clean.size ); + vs.var( 'size', calcSize(clean.size)); } // focus point to score by distance diff --git a/query/search.js b/query/search.js index da7773a9..64acff3f 100644 --- a/query/search.js +++ b/query/search.js @@ -2,7 +2,8 @@ var peliasQuery = require('pelias-query'), defaults = require('./search_defaults'), textParser = require('./text_parser'), check = require('check-types'), - geolib = require('geolib'); + geolib = require('geolib'), + calcSize = require('../helper/sizeCalculator'); //------------------------------ // general-purpose search query @@ -52,9 +53,8 @@ function generateQuery( clean ){ vs.var( 'input:name', clean.text ); // size - if( clean.size ){ - vs.var( 'size', clean.size ); - } + // specify twice as much data as we need so we can filter out dupes + vs.var( 'size', calcSize(clean.size || defaults.size)); // focus point if( check.number(clean['focus.point.lat']) && diff --git a/routes/v1.js b/routes/v1.js index 4d112458..9db30934 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -30,6 +30,7 @@ var postProc = { distances: require('../middleware/distance'), confidenceScores: require('../middleware/confidenceScore'), confidenceScoresReverse: require('../middleware/confidenceScoreReverse'), + dedupe: require('../middleware/dedupe'), localNamingConventions: require('../middleware/localNamingConventions'), renamePlacenames: require('../middleware/renamePlacenames'), geocodeJSON: require('../middleware/geocodeJSON'), @@ -61,6 +62,7 @@ function addRoutes(app, peliasConfig) { controllers.search(), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), @@ -72,6 +74,7 @@ function addRoutes(app, peliasConfig) { controllers.search(null, require('../query/autocomplete')), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), @@ -85,6 +88,7 @@ function addRoutes(app, peliasConfig) { // reverse confidence scoring depends on distance from origin // so it must be calculated first postProc.confidenceScoresReverse(), + postProc.dedupe(), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.geocodeJSON(peliasConfig, base), diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 8d0e9914..117238c7 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -115,6 +115,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 8300b026..3d8da814 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -115,6 +115,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 53ece28e..5a8cbc3e 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -87,6 +87,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js new file mode 100644 index 00000000..ae048564 --- /dev/null +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -0,0 +1,248 @@ +module.exports = [ + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.039265 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.23246, + 'lat': 39.99288 + }, + 'address': {}, + 'local_admin': 'West Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'Lampeter-Strasburg High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Lampeter', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Wheatland Mills', + 'category': [ + 'education' + ], + '_id': '4559068', + '_type': 'geoname', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.20746, + 'lat': 40.03927 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '5187980', + '_type': 'geoname', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.232457, + 'lat': 39.992877 + }, + 'address': {}, + 'local_admin': 'West Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'Lampeter-Strasburg High School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Lampeter', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Wheatland Mills', + 'category': [ + 'education' + ], + '_id': '357294404', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.038987 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357283977', + '_type': 'osmnode', + '_score': 1.1036991, + 'confidence': 0.664 + }, + { + 'center_point': { + 'lon': -76.20746, + 'lat': 40.03899 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '5187966', + '_type': 'geoname', + '_score': 1.1036991, + 'confidence': 0.664 + }, + { + 'center_point': { + 'lon': -94.167445, + 'lat': 38.762788 + }, + 'address': {}, + 'local_admin': 'Polk', + 'admin1_abbr': 'MO', + 'name': { + 'default': 'Strasburg School' + }, + 'admin1': 'Missouri', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Cass County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '358058986', + '_type': 'osmnode', + '_score': 1.0492544, + 'confidence': 0.658 + }, + { + 'center_point': { + 'lon': -78.36317, + 'lat': 38.98445 + }, + 'address': {}, + 'admin1_abbr': 'VA', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'Virginia', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Shenandoah County', + 'admin0': 'United States', + 'neighborhood': 'Strasburg Junction', + 'category': [ + 'education' + ], + '_id': '4787978', + '_type': 'geoname', + '_score': 0.9724125, + 'confidence': 0.649 + }, + { + 'center_point': { + 'lon': -100.16516, + 'lat': 46.13427 + }, + 'address': {}, + 'local_admin': 'Strasburg', + 'admin1_abbr': 'ND', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'North Dakota', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Emmons County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '9683163', + '_type': 'geoname', + '_score': 0.9724125, + 'confidence': 0.649 + }, + { + 'center_point': { + 'lon': -81.532392, + 'lat': 40.597578 + }, + 'address': {}, + 'local_admin': 'Franklin', + 'admin1_abbr': 'OH', + 'name': { + 'default': 'Strasburg High School' + }, + 'admin1': 'Ohio', + 'locality': 'Strasburg', + 'alpha3': 'USA', + 'admin2': 'Tuscarawas County', + 'admin0': 'United States', + 'category': [ + 'education' + ], + '_id': '356646971', + '_type': 'osmway', + '_score': 0.9724125, + 'confidence': 0.649 + } +]; \ No newline at end of file diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index 3e6f83bc..a4e46bd8 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -97,6 +97,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 93572705..34130fe4 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -178,7 +178,7 @@ module.exports = { } } }, - 'size': 10, + 'size': 20, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js index c9859aeb..910a3d13 100644 --- a/test/unit/fixture/search_linguistic_bbox.js +++ b/test/unit/fixture/search_linguistic_bbox.js @@ -103,6 +103,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js index 562f32c3..8140fe66 100644 --- a/test/unit/fixture/search_linguistic_focus.js +++ b/test/unit/fixture/search_linguistic_focus.js @@ -117,6 +117,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js index bd7c40ab..7b94dd27 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox.js +++ b/test/unit/fixture/search_linguistic_focus_bbox.js @@ -133,6 +133,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js index cf8b1625..8602dddf 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island.js +++ b/test/unit/fixture/search_linguistic_focus_null_island.js @@ -117,6 +117,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js index 53ece28e..5a8cbc3e 100644 --- a/test/unit/fixture/search_linguistic_only.js +++ b/test/unit/fixture/search_linguistic_only.js @@ -87,6 +87,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 10, + 'size': 20, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js index 46cdc3e9..70ce42a0 100644 --- a/test/unit/fixture/search_linguistic_viewport.js +++ b/test/unit/fixture/search_linguistic_viewport.js @@ -128,7 +128,7 @@ module.exports = { } } }, - 'size': 10, + 'size': 20, 'track_scores': true, 'sort': [ '_score' diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js index c5306919..f6ff4cda 100644 --- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js +++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js @@ -128,7 +128,7 @@ module.exports = { } } }, - 'size': 10, + 'size': 20, 'track_scores': true, 'sort': [ '_score' diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index 6e0fbd5b..4a21a625 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -145,7 +145,7 @@ module.exports = { } } }, - 'size': 10, + 'size': 20, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index cc04943b..ada8b830 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -161,7 +161,7 @@ module.exports = { } } }, - 'size': 10, + 'size': 20, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js new file mode 100644 index 00000000..2beedaef --- /dev/null +++ b/test/unit/helper/sizeCalculator.js @@ -0,0 +1,39 @@ + +var calcSize = require('../../../helper/sizeCalculator.js'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('interface', function(t) { + t.equal(typeof calcSize, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.tests.valid = function(test, common) { + test('size=0', function (t) { + t.equal(calcSize(0), 1); + t.end(); + }); + + test('size=1', function (t) { + t.equal(calcSize(1), 1); + t.end(); + }); + + test('size=10', function (t) { + t.equal(calcSize(10), 20); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('sizeCalculator: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js new file mode 100644 index 00000000..31d4bf6f --- /dev/null +++ b/test/unit/middleware/dedupe.js @@ -0,0 +1,52 @@ +var data = require('../fixture/dedupe_elasticsearch_results'); +var dedupe = require('../../../middleware/dedupe')(); + +module.exports.tests = {}; + +module.exports.tests.dedupe = function(test, common) { + test('filter out duplicates', function(t) { + var req = { + clean: { + text: 'lampeter strasburg high school', + size: 100 + } + }; + var res = { + data: data + }; + + var expectedCount = 7; + dedupe(req, res, function () { + t.equal(res.data.length, expectedCount, 'results have fewer items than before'); + t.end(); + }); + }); + + test('truncate results based on specified size', function(t) { + var req = { + clean: { + text: 'lampeter strasburg high school', + size: 3 + } + }; + var res = { + data: data + }; + + dedupe(req, res, function () { + t.equal(res.data.length, req.clean.size, 'results have fewer items than before'); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('[middleware] dedupe: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js index 7a2f9d07..5f6e39ca 100644 --- a/test/unit/query/reverse.js +++ b/test/unit/query/reverse.js @@ -80,13 +80,14 @@ module.exports.tests.query = function(test, common) { test('size fuzz test', function(t) { // test different sizes var sizes = [1,2,10,undefined,null]; - sizes.forEach( function( size ){ + var expectedSizes = [1,4,20,1,1]; + sizes.forEach( function( size, index ){ var query = generate({ 'point.lat': 29.49136, 'point.lon': -82.50622, size: size }); var compiled = JSON.parse( JSON.stringify( query ) ); - t.equal( compiled.size, size ? size : 1, 'valid reverse query for size: '+ size); + t.equal( compiled.size, expectedSizes[index], 'valid reverse query for size: '+ size); }); t.end(); }); diff --git a/test/unit/run.js b/test/unit/run.js index 191dcf9c..0510cda6 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -12,10 +12,12 @@ var tests = [ require('./helper/text_parser'), require('./helper/type_mapping'), require('./helper/types'), + require('./helper/sizeCalculator'), require('./middleware/confidenceScore'), require('./middleware/confidenceScoreReverse'), require('./middleware/distance'), require('./middleware/localNamingConventions'), + require('./middleware/dedupe'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'), From 42d940f8c85d491d48d7aa393c2c60f0d322786d Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Wed, 2 Dec 2015 16:47:49 -0500 Subject: [PATCH 2/3] Add simple normalizer (lowercase + remove punctuation) --- controller/search.js | 2 +- middleware/dedupe.js | 29 ++++++++++++++----- .../fixture/dedupe_elasticsearch_results.js | 25 ++++++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/controller/search.js b/controller/search.js index 38bfb481..309e7647 100644 --- a/controller/search.js +++ b/controller/search.js @@ -16,7 +16,7 @@ function setup( backend, query ){ } // log clean parameters for stats - logger.info(req.clean); + logger.info('[req]', 'endpoint=' + req.path, req.clean); // backend command var cmd = { diff --git a/middleware/dedupe.js b/middleware/dedupe.js index c6e95c1c..445190c2 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -1,5 +1,4 @@ -var util = require('util'); -var logger = require('pelias-logger').get('api:middle:dedupe'); +var logger = require('pelias-logger').get('api'); var _ = require('lodash'); function setup() { @@ -45,8 +44,8 @@ function isDifferent(item1, item2) { if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) { propMatch(item1.name, item2.name, 'default'); } - else if (item1.name !== item2.name) { - throw 'different'; + else { + propMatch(item1, item2, 'name'); } if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) { @@ -68,15 +67,29 @@ function isDifferent(item1, item2) { /** * Throw exception if properties are different * - * @param item1 - * @param item2 - * @param prop + * @param {object} item1 + * @param {object} item2 + * @param {string} prop + * @throws {string} */ function propMatch(item1, item2, prop) { - if (item1[prop] !== item2[prop]) { + if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) { throw 'different'; } } +/** + * Remove punctuation and lowercase + * + * @param {string} str + * @returns {string} + */ +function normalizeString(str) { + if (!str) { + return ''; + } + return _.words(str.toLowerCase()).join(' '); +} + module.exports = setup; \ No newline at end of file diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js index ae048564..6ed761c4 100644 --- a/test/unit/fixture/dedupe_elasticsearch_results.js +++ b/test/unit/fixture/dedupe_elasticsearch_results.js @@ -24,6 +24,31 @@ module.exports = [ '_score': 1.2367082, 'confidence': 0.879 }, + { + 'center_point': { + 'lon': -76.207456, + 'lat': 40.039265 + }, + 'address': {}, + 'local_admin': 'East Lampeter', + 'admin1_abbr': 'PA', + 'name': { + 'default': 'East Lampeter, High-School' + }, + 'admin1': 'Pennsylvania', + 'locality': 'Smoketown', + 'alpha3': 'USA', + 'admin2': 'Lancaster County', + 'admin0': 'United States', + 'neighborhood': 'Greenland', + 'category': [ + 'education' + ], + '_id': '357321757', + '_type': 'osmnode', + '_score': 1.2367082, + 'confidence': 0.879 + }, { 'center_point': { 'lon': -76.23246, From 9fa5fc5a7717d0ae0fe692cd4199c008cee8a10e Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Fri, 4 Dec 2015 11:35:34 -0500 Subject: [PATCH 3/3] calcSize became middleware (exposed and fixed bug in query defaults) --- helper/sizeCalculator.js | 21 -------- middleware/dedupe.js | 12 +++-- middleware/sizeCalculator.js | 35 +++++++++++++ query/autocomplete.js | 6 +-- query/autocomplete_defaults.js | 8 +-- query/reverse.js | 13 ++--- query/reverse_defaults.js | 10 ++-- query/search.js | 8 +-- query/search_defaults.js | 8 +-- routes/v1.js | 5 +- .../fixture/autocomplete_linguistic_focus.js | 3 +- ...tocomplete_linguistic_focus_null_island.js | 3 +- .../fixture/autocomplete_linguistic_only.js | 3 +- test/unit/fixture/reverse_boundary_circle.js | 5 +- test/unit/fixture/reverse_null_island.js | 3 +- test/unit/fixture/reverse_standard.js | 3 +- .../fixture/reverse_with_boundary_country.js | 3 +- test/unit/fixture/search_boundary_country.js | 2 +- test/unit/fixture/search_full_address.js | 50 +++++++++---------- test/unit/fixture/search_linguistic_bbox.js | 2 +- test/unit/fixture/search_linguistic_focus.js | 2 +- .../fixture/search_linguistic_focus_bbox.js | 2 +- .../search_linguistic_focus_null_island.js | 2 +- test/unit/fixture/search_linguistic_only.js | 2 +- .../fixture/search_linguistic_viewport.js | 2 +- ...search_linguistic_viewport_min_diagonal.js | 2 +- test/unit/fixture/search_partial_address.js | 33 ++++++------ test/unit/fixture/search_regions_address.js | 41 ++++++++------- test/unit/helper/sizeCalculator.js | 42 +++++++++++++--- test/unit/query/reverse.js | 8 +-- test/unit/query/search.js | 22 ++++---- 31 files changed, 201 insertions(+), 160 deletions(-) delete mode 100644 helper/sizeCalculator.js create mode 100644 middleware/sizeCalculator.js diff --git a/helper/sizeCalculator.js b/helper/sizeCalculator.js deleted file mode 100644 index b51bed76..00000000 --- a/helper/sizeCalculator.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Utility for calculating query result size - * incorporating padding for dedupe process - */ - -var SIZE_PADDING = 2; - -/** - * Add padding or set to 1 - * - * @param {number} cleanSize - * @returns {number} - */ -module.exports = function calculateSize(cleanSize) { - switch (cleanSize || 1) { - case 1: - return 1; - default: - return cleanSize * SIZE_PADDING; - } -}; \ No newline at end of file diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 445190c2..b02ca2a3 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -35,6 +35,7 @@ function dedupeResults(req, res, next) { * @param {object} item1 * @param {object} item2 * @returns {boolean} + * @throws {Error} */ function isDifferent(item1, item2) { try { @@ -54,11 +55,14 @@ function isDifferent(item1, item2) { propMatch(item1.address, item2.address, 'zip'); } else if (item1.address !== item2.address) { - throw 'different'; + throw new Error('different'); } } catch (err) { - return true; + if (err.message === 'different') { + return true; + } + throw err; } return false; @@ -70,11 +74,11 @@ function isDifferent(item1, item2) { * @param {object} item1 * @param {object} item2 * @param {string} prop - * @throws {string} + * @throws {Error} */ function propMatch(item1, item2, prop) { if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) { - throw 'different'; + throw new Error('different'); } } diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js new file mode 100644 index 00000000..88334d3b --- /dev/null +++ b/middleware/sizeCalculator.js @@ -0,0 +1,35 @@ +var _ = require('lodash'); + +var SIZE_PADDING = 2; + +/** + * Utility for calculating query result size + * incorporating padding for dedupe process + */ +function setup() { + return function setQuerySize(req, res, next) { + if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { + return next(); + } + + req.clean.querySize = calculateSize(req.clean.size); + next(); + }; +} + +/** + * Add padding or set to 1 + * + * @param {number} cleanSize + * @returns {number} + */ +function calculateSize(cleanSize) { + switch (cleanSize || 1) { + case 1: + return 1; + default: + return cleanSize * SIZE_PADDING; + } +} + +module.exports = setup; \ No newline at end of file diff --git a/query/autocomplete.js b/query/autocomplete.js index d9f3f3be..ca4acdcb 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -1,8 +1,7 @@ var peliasQuery = require('pelias-query'), defaults = require('./autocomplete_defaults'), - check = require('check-types'), - calcSize = require('../helper/sizeCalculator'); + check = require('check-types'); //------------------------------ // autocomplete query @@ -31,9 +30,6 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); - // always 10 (not user definable due to caching) - vs.var( 'size', calcSize(10)); - // focus point if( check.number(clean['focus.point.lat']) && check.number(clean['focus.point.lon']) ){ diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 401ebf73..28cf6e7a 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 20, 'track_scores': true, 'centroid:field': 'center_point', diff --git a/query/reverse.js b/query/reverse.js index 5f196cbc..856b9540 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,7 +1,6 @@ var peliasQuery = require('pelias-query'), defaults = require('./reverse_defaults'), - check = require('check-types'), - calcSize = require('../helper/sizeCalculator'); + check = require('check-types'); //------------------------------ // reverse geocode query @@ -23,15 +22,9 @@ function generateQuery( clean ){ var vs = new peliasQuery.Vars( defaults ); - // set defaults - vs.set({ - 'size': 1, - 'boundary:circle:radius': '500km' - }); - // set size - if( clean.size ){ - vs.var( 'size', calcSize(clean.size)); + if( clean.querySize ){ + vs.var( 'size', clean.querySize); } // focus point to score by distance diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index f6a9638c..01e74773 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 1, 'track_scores': true, 'centroid:field': 'center_point', @@ -12,7 +12,7 @@ module.exports = extend( false, peliasQuery.defaults, { 'sort:distance:order': 'asc', 'sort:distance:distance_type': 'plane', - 'boundary:circle:radius': '50km', + 'boundary:circle:radius': '500km', 'boundary:circle:distance_type': 'plane', 'boundary:circle:optimize_bbox': 'indexed', 'boundary:circle:_cache': true, diff --git a/query/search.js b/query/search.js index 64acff3f..91999b1e 100644 --- a/query/search.js +++ b/query/search.js @@ -2,8 +2,7 @@ var peliasQuery = require('pelias-query'), defaults = require('./search_defaults'), textParser = require('./text_parser'), check = require('check-types'), - geolib = require('geolib'), - calcSize = require('../helper/sizeCalculator'); + geolib = require('geolib'); //------------------------------ // general-purpose search query @@ -53,8 +52,9 @@ function generateQuery( clean ){ vs.var( 'input:name', clean.text ); // size - // specify twice as much data as we need so we can filter out dupes - vs.var( 'size', calcSize(clean.size || defaults.size)); + if( clean.querySize ) { + vs.var( 'size', clean.querySize ); + } // focus point if( check.number(clean['focus.point.lat']) && diff --git a/query/search_defaults.js b/query/search_defaults.js index 401ebf73..28cf6e7a 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -1,10 +1,10 @@ -var peliasQuery = require('pelias-query'), - extend = require('extend'); +var peliasQuery = require('pelias-query'); +var _ = require('lodash'); -module.exports = extend( false, peliasQuery.defaults, { +module.exports = _.merge({}, peliasQuery.defaults, { - 'size': 10, + 'size': 20, 'track_scores': true, 'centroid:field': 'center_point', diff --git a/routes/v1.js b/routes/v1.js index 9db30934..49c9ea4c 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -12,7 +12,8 @@ var sanitisers = { /** ----------------------- middleware ------------------------ **/ var middleware = { - types: require('../middleware/_types') + types: require('../middleware/_types'), + calcSize: require('../middleware/sizeCalculator') }; /** ----------------------- controllers ----------------------- **/ @@ -59,6 +60,7 @@ function addRoutes(app, peliasConfig) { search: createRouter([ sanitisers.search.middleware, middleware.types, + middleware.calcSize(), controllers.search(), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), @@ -83,6 +85,7 @@ function addRoutes(app, peliasConfig) { reverse: createRouter([ sanitisers.reverse.middleware, middleware.types, + middleware.calcSize(), controllers.search(undefined, reverseQuery), postProc.distances('point.'), // reverse confidence scoring depends on distance from origin diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 117238c7..0ef70786 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -115,6 +116,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 3d8da814..59b72061 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -115,6 +116,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 5a8cbc3e..e20170c5 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/autocomplete_defaults'); module.exports = { 'query': { @@ -87,6 +88,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/reverse_boundary_circle.js b/test/unit/fixture/reverse_boundary_circle.js index b7cc058f..897f019f 100644 --- a/test/unit/fixture/reverse_boundary_circle.js +++ b/test/unit/fixture/reverse_boundary_circle.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -10,7 +11,7 @@ module.exports = { 'must': [ { 'geo_distance': { - 'distance': '500km', + 'distance': vs.distance, 'distance_type': 'plane', 'optimize_bbox': 'indexed', '_cache': true, @@ -38,6 +39,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; \ No newline at end of file diff --git a/test/unit/fixture/reverse_null_island.js b/test/unit/fixture/reverse_null_island.js index f29ab293..5f020ad2 100644 --- a/test/unit/fixture/reverse_null_island.js +++ b/test/unit/fixture/reverse_null_island.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -40,6 +41,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/reverse_standard.js b/test/unit/fixture/reverse_standard.js index 08a005e6..d0b0b734 100644 --- a/test/unit/fixture/reverse_standard.js +++ b/test/unit/fixture/reverse_standard.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -40,6 +41,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/reverse_with_boundary_country.js b/test/unit/fixture/reverse_with_boundary_country.js index f29569b2..8d739eaa 100644 --- a/test/unit/fixture/reverse_with_boundary_country.js +++ b/test/unit/fixture/reverse_with_boundary_country.js @@ -1,3 +1,4 @@ +var vs = require('../../../query/reverse_defaults'); module.exports = { 'query': { @@ -49,6 +50,6 @@ module.exports = { } } ], - 'size': 1, + 'size': vs.size, 'track_scores': true }; diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index a4e46bd8..3e6f83bc 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -97,6 +97,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 34130fe4..c30896e3 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -1,6 +1,4 @@ - -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -89,88 +87,88 @@ module.exports = { 'match': { 'address.number': { 'query': 123, - 'boost': vs.var('address:housenumber:boost').get(), - 'analyzer': vs.var('address:housenumber:analyzer').get() + 'boost': vs['address:housenumber:boost'], + 'analyzer': vs['address:housenumber:analyzer'] } } }, { 'match': { 'address.street': { 'query': 'main st', - 'boost': vs.var('address:street:boost').get(), - 'analyzer': vs.var('address:street:analyzer').get() + 'boost': vs['address:street:boost'], + 'analyzer': vs['address:street:analyzer'] } } }, { 'match': { 'address.zip': { 'query': 10010, - 'boost': vs.var('address:postcode:boost').get(), - 'analyzer': vs.var('address:postcode:analyzer').get() + 'boost': vs['address:postcode:boost'], + 'analyzer': vs['address:postcode:analyzer'] } } }, { 'match': { 'alpha3': { 'query': 'USA', - 'boost': vs.var('admin:alpha3:boost').get(), - 'analyzer': vs.var('admin:alpha3:analyzer').get() + 'boost': vs['admin:alpha3:boost'], + 'analyzer': vs['admin:alpha3:analyzer'] } } }, { 'match': { 'admin0': { 'query': 'new york', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'new york', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'NY', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'new york', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'new york', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'new york', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'new york', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] @@ -178,7 +176,7 @@ module.exports = { } } }, - 'size': 20, + 'size': 10, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js index 910a3d13..c9859aeb 100644 --- a/test/unit/fixture/search_linguistic_bbox.js +++ b/test/unit/fixture/search_linguistic_bbox.js @@ -103,6 +103,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js index 8140fe66..562f32c3 100644 --- a/test/unit/fixture/search_linguistic_focus.js +++ b/test/unit/fixture/search_linguistic_focus.js @@ -117,6 +117,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js index 7b94dd27..bd7c40ab 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox.js +++ b/test/unit/fixture/search_linguistic_focus_bbox.js @@ -133,6 +133,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js index 8602dddf..cf8b1625 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island.js +++ b/test/unit/fixture/search_linguistic_focus_null_island.js @@ -117,6 +117,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js index 5a8cbc3e..53ece28e 100644 --- a/test/unit/fixture/search_linguistic_only.js +++ b/test/unit/fixture/search_linguistic_only.js @@ -87,6 +87,6 @@ module.exports = { } }, 'sort': [ '_score' ], - 'size': 20, + 'size': 10, 'track_scores': true }; diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js index 70ce42a0..46cdc3e9 100644 --- a/test/unit/fixture/search_linguistic_viewport.js +++ b/test/unit/fixture/search_linguistic_viewport.js @@ -128,7 +128,7 @@ module.exports = { } } }, - 'size': 20, + 'size': 10, 'track_scores': true, 'sort': [ '_score' diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js index f6ff4cda..c5306919 100644 --- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js +++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js @@ -128,7 +128,7 @@ module.exports = { } } }, - 'size': 20, + 'size': 10, 'track_scores': true, 'sort': [ '_score' diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index 4a21a625..745598da 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -1,6 +1,5 @@ -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -88,56 +87,56 @@ module.exports = { 'match': { 'admin0': { 'query': 'new york', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'new york', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'new york', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'new york', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'new york', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'new york', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'new york', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] @@ -145,7 +144,7 @@ module.exports = { } } }, - 'size': 20, + 'size': 10, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index ada8b830..2a06b6a9 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -1,6 +1,5 @@ -var peliasQuery = require('pelias-query'), - vs = new peliasQuery.Vars( peliasQuery.defaults ); +var vs = require('../../../query/search_defaults'); module.exports = { 'query': { @@ -88,72 +87,72 @@ module.exports = { 'match': { 'address.number': { 'query': 1, - 'boost': vs.var('address:housenumber:boost').get(), - 'analyzer': vs.var('address:housenumber:analyzer').get() + 'boost': vs['address:housenumber:boost'], + 'analyzer': vs['address:housenumber:analyzer'] } } }, { 'match': { 'address.street': { 'query': 'water st', - 'boost': vs.var('address:street:boost').get(), - 'analyzer': vs.var('address:street:analyzer').get() + 'boost': vs['address:street:boost'], + 'analyzer': vs['address:street:analyzer'] } } }, { 'match': { 'admin0': { 'query': 'manhattan', - 'boost': vs.var('admin:admin0:boost').get(), - 'analyzer': vs.var('admin:admin0:analyzer').get() + 'boost': vs['admin:admin0:boost'], + 'analyzer': vs['admin:admin0:analyzer'] } } }, { 'match': { 'admin1': { 'query': 'manhattan', - 'boost': vs.var('admin:admin1:boost').get(), - 'analyzer': vs.var('admin:admin1:analyzer').get() + 'boost': vs['admin:admin1:boost'], + 'analyzer': vs['admin:admin1:analyzer'] } } }, { 'match': { 'admin1_abbr': { 'query': 'NY', - 'boost': vs.var('admin:admin1_abbr:boost').get(), - 'analyzer': vs.var('admin:admin1_abbr:analyzer').get() + 'boost': vs['admin:admin1_abbr:boost'], + 'analyzer': vs['admin:admin1_abbr:analyzer'] } } }, { 'match': { 'admin2': { 'query': 'manhattan', - 'boost': vs.var('admin:admin2:boost').get(), - 'analyzer': vs.var('admin:admin2:analyzer').get() + 'boost': vs['admin:admin2:boost'], + 'analyzer': vs['admin:admin2:analyzer'] } } }, { 'match': { 'local_admin': { 'query': 'manhattan', - 'boost': vs.var('admin:local_admin:boost').get(), - 'analyzer': vs.var('admin:local_admin:analyzer').get() + 'boost': vs['admin:local_admin:boost'], + 'analyzer': vs['admin:local_admin:analyzer'] } } }, { 'match': { 'locality': { 'query': 'manhattan', - 'boost': vs.var('admin:locality:boost').get(), - 'analyzer': vs.var('admin:locality:analyzer').get() + 'boost': vs['admin:locality:boost'], + 'analyzer': vs['admin:locality:analyzer'] } } }, { 'match': { 'neighborhood': { 'query': 'manhattan', - 'boost': vs.var('admin:neighborhood:boost').get(), - 'analyzer': vs.var('admin:neighborhood:analyzer').get() + 'boost': vs['admin:neighborhood:boost'], + 'analyzer': vs['admin:neighborhood:analyzer'] } } }] @@ -161,7 +160,7 @@ module.exports = { } } }, - 'size': 20, + 'size': 10, 'sort': [ '_score' ], 'track_scores': true }; diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index 2beedaef..41d854d4 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -1,5 +1,5 @@ -var calcSize = require('../../../helper/sizeCalculator.js'); +var calcSize = require('../../../middleware/sizeCalculator.js')(); module.exports.tests = {}; @@ -11,19 +11,47 @@ module.exports.tests.interface = function(test, common) { }; module.exports.tests.valid = function(test, common) { + var req = { clean: {} }; + function setup(val) { + if (isNaN(val)) { + delete req.clean.size; + } + else { + req.clean.size = val; + } + delete req.clean.querySize; + } + test('size=0', function (t) { - t.equal(calcSize(0), 1); - t.end(); + setup(0); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 1); + t.end(); + }); }); test('size=1', function (t) { - t.equal(calcSize(1), 1); - t.end(); + setup(1); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 1); + t.end(); + }); }); test('size=10', function (t) { - t.equal(calcSize(10), 20); - t.end(); + setup(10); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 20); + t.end(); + }); + }); + + test('no size', function (t) { + setup(); + calcSize(req, {}, function () { + t.equal(req.clean.hasOwnProperty('querySize'), false); + t.end(); + }); }); }; diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js index 5f6e39ca..7e6c2ba1 100644 --- a/test/unit/query/reverse.js +++ b/test/unit/query/reverse.js @@ -79,15 +79,15 @@ module.exports.tests.query = function(test, common) { test('size fuzz test', function(t) { // test different sizes - var sizes = [1,2,10,undefined,null]; - var expectedSizes = [1,4,20,1,1]; + var sizes = [1,4,20,undefined,null]; + var expected = [1,4,20,1,1]; sizes.forEach( function( size, index ){ var query = generate({ - 'point.lat': 29.49136, 'point.lon': -82.50622, size: size + 'point.lat': 29.49136, 'point.lon': -82.50622, querySize: size }); var compiled = JSON.parse( JSON.stringify( query ) ); - t.equal( compiled.size, expectedSizes[index], 'valid reverse query for size: '+ size); + t.equal( compiled.size, expected[index], 'valid reverse query for size: '+ size); }); t.end(); }); diff --git a/test/unit/query/search.js b/test/unit/query/search.js index 15527222..7fa81979 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.query = function(test, common) { test('valid search + focus + bbox', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, 'boundary.rect.min_lat': 47.47, 'boundary.rect.max_lon': -61.84, @@ -31,7 +31,7 @@ module.exports.tests.query = function(test, common) { test('valid search + bbox', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'boundary.rect.min_lat': 47.47, 'boundary.rect.max_lon': -61.84, 'boundary.rect.max_lat': 11.51, @@ -48,7 +48,7 @@ module.exports.tests.query = function(test, common) { test('valid lingustic-only search', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, layers: ['test'] }); @@ -61,7 +61,7 @@ module.exports.tests.query = function(test, common) { test('search search + focus', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, layers: ['test'] }); @@ -75,7 +75,7 @@ module.exports.tests.query = function(test, common) { test('search search + viewport', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.viewport.min_lat': 28.49136, 'focus.viewport.max_lat': 30.49136, 'focus.viewport.min_lon': -87.50622, @@ -92,7 +92,7 @@ module.exports.tests.query = function(test, common) { test('search with viewport diagonal < 1km should set scale to 1km', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.viewport.min_lat': 28.49135, 'focus.viewport.max_lat': 28.49137, 'focus.viewport.min_lon': -87.50622, @@ -109,7 +109,7 @@ module.exports.tests.query = function(test, common) { test('search search + focus on null island', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, 'focus.point.lat': 0, 'focus.point.lon': 0, layers: ['test'] }); @@ -126,7 +126,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(address), }); @@ -142,7 +142,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); @@ -158,7 +158,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], - size: 10, + querySize: 10, parsed_text: parser.get_parsed_address(partial_address), }); @@ -171,7 +171,7 @@ module.exports.tests.query = function(test, common) { test('valid boundary.country search', function(t) { var query = generate({ - text: 'test', size: 10, + text: 'test', querySize: 10, layers: ['test'], 'boundary.country': 'ABC' });