From d03a8c45867cd5661486c657b3850b2b5eaaa3fa Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 28 Oct 2016 15:30:44 -0400 Subject: [PATCH] added support for component geocoding - created `/component` route - broke out trimByGranularityComponent but could conceivably be combined with existing - added `address` support to text_parser - added `component` sanitizer wrapper --- middleware/trimByGranularityComponent.js | 74 +++ query/component.js | 117 +++++ query/text_parser.js | 4 + routes/v1.js | 28 +- sanitizer/component.js | 26 ++ .../middleware/trimByGranularityComponent.js | 431 ++++++++++++++++++ test/unit/query/component.js | 283 ++++++++++++ test/unit/query/text_parser.js | 3 + test/unit/run.js | 2 + test/unit/sanitizer/component.js | 113 +++++ 10 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 middleware/trimByGranularityComponent.js create mode 100644 query/component.js create mode 100644 sanitizer/component.js create mode 100644 test/unit/middleware/trimByGranularityComponent.js create mode 100644 test/unit/query/component.js create mode 100644 test/unit/sanitizer/component.js diff --git a/middleware/trimByGranularityComponent.js b/middleware/trimByGranularityComponent.js new file mode 100644 index 00000000..25440598 --- /dev/null +++ b/middleware/trimByGranularityComponent.js @@ -0,0 +1,74 @@ +var _ = require('lodash'); + +// This middleware component trims the results array by granularity when +// FallbackQuery was used. FallbackQuery is used for inputs like +// `1090 N Charlotte St, Lancaster, PA` where the address may not exist and +// we must fall back to trying `Lancaster, PA`. If the address does exist then +// FallbackQuery will return results for: +// - address+city+state +// - city+state +// - state +// +// Because the address matched, we're not interested in city+state or state, so +// this component removes results that aren't the most granular. + +// layers in increasing order of granularity +var layers = [ + 'venue', + 'street', + 'address', + 'neighbourhood', + 'borough', + 'locality', + 'localadmin', + 'county', + 'macrocounty', + 'region', + 'macroregion', + 'dependency', + 'country' +]; + +// this helper method returns `true` if every result has a matched_query +// starting with `fallback.` +function isFallbackQuery(results) { + return results.every(function(result) { + return result.hasOwnProperty('_matched_queries') && + !_.isEmpty(result._matched_queries) && + _.startsWith(result._matched_queries[0], 'fallback.'); + }); +} + +function hasRecordsAtLayers(results, layer) { + return results.some(function(result) { + return result._matched_queries[0] === 'fallback.' + layer; + }); +} + +function retainRecordsAtLayers(results, layer) { + return results.filter(function(result) { + return result._matched_queries[0] === 'fallback.' + layer; + }); +} + +function setup() { + return function trim(req, res, next) { + // don't do anything if there are no results or there are non-fallback.* named queries + // there should never be a mixture of fallback.* and non-fallback.* named queries + if (_.isUndefined(res.data) || !isFallbackQuery(res.data)) { + return next(); + } + + // start at the most granular possible layer. if there are results at a layer + // then remove everything not at that layer. + layers.forEach(function(layer) { + if (hasRecordsAtLayers(res.data, layer )) { + res.data = retainRecordsAtLayers(res.data, layer); + } + }); + + next(); + }; +} + +module.exports = setup; diff --git a/query/component.js b/query/component.js new file mode 100644 index 00000000..5266d8c1 --- /dev/null +++ b/query/component.js @@ -0,0 +1,117 @@ +var peliasQuery = require('pelias-query'), + defaults = require('./search_defaults'), + textParser = require('./text_parser'), + check = require('check-types'); + +//------------------------------ +// general-purpose search query +//------------------------------ +var componentQuery = new peliasQuery.layout.ComponentFallbackQuery(); + +// scoring boost +componentQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) ); +componentQuery.score( peliasQuery.view.popularity_only_function ); +componentQuery.score( peliasQuery.view.population_only_function ); +// -------------------------------- + +// non-scoring hard filters +componentQuery.filter( peliasQuery.view.boundary_country ); +componentQuery.filter( peliasQuery.view.boundary_circle ); +componentQuery.filter( peliasQuery.view.boundary_rect ); +componentQuery.filter( peliasQuery.view.sources ); +componentQuery.filter( peliasQuery.view.layers ); +componentQuery.filter( peliasQuery.view.categories ); +// -------------------------------- + +/** + map request variables to query variables for all inputs + provided by this HTTP request. +**/ +function generateQuery( clean ){ + + var vs = new peliasQuery.Vars( defaults ); + + // input text + vs.var( 'input:name', clean.text ); + + // sources + vs.var( 'sources', clean.sources); + + // layers + vs.var( 'layers', clean.layers); + + // categories + if (clean.categories) { + vs.var('input:categories', clean.categories); + } + + // size + if( clean.querySize ) { + vs.var( 'size', clean.querySize ); + } + + // focus point + if( check.number(clean['focus.point.lat']) && + check.number(clean['focus.point.lon']) ){ + vs.set({ + 'focus:point:lat': clean['focus.point.lat'], + 'focus:point:lon': clean['focus.point.lon'] + }); + } + + // boundary rect + if( check.number(clean['boundary.rect.min_lat']) && + check.number(clean['boundary.rect.max_lat']) && + check.number(clean['boundary.rect.min_lon']) && + check.number(clean['boundary.rect.max_lon']) ){ + vs.set({ + 'boundary:rect:top': clean['boundary.rect.max_lat'], + 'boundary:rect:right': clean['boundary.rect.max_lon'], + 'boundary:rect:bottom': clean['boundary.rect.min_lat'], + 'boundary:rect:left': clean['boundary.rect.min_lon'] + }); + } + + // boundary circle + // @todo: change these to the correct request variable names + if( check.number(clean['boundary.circle.lat']) && + check.number(clean['boundary.circle.lon']) ){ + vs.set({ + 'boundary:circle:lat': clean['boundary.circle.lat'], + 'boundary:circle:lon': clean['boundary.circle.lon'] + }); + + if( check.number(clean['boundary.circle.radius']) ){ + vs.set({ + 'boundary:circle:radius': Math.round( clean['boundary.circle.radius'] ) + 'km' + }); + } + } + + // boundary country + if( check.string(clean['boundary.country']) ){ + vs.set({ + 'boundary:country': clean['boundary.country'] + }); + } + + // run the address parser + if( clean.parsed_text ){ + textParser( clean.parsed_text, vs ); + } + + var q = getQuery(vs); + + console.log(JSON.stringify(q.body, null, 2)); + + return q; +} + +function getQuery(vs) { + return { + type: 'fallback', + body: componentQuery.render(vs) + }; +} + +module.exports = generateQuery; diff --git a/query/text_parser.js b/query/text_parser.js index c5b8da44..4ccc8661 100644 --- a/query/text_parser.js +++ b/query/text_parser.js @@ -14,6 +14,10 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ vs.var('input:category', parsed_text.category); } + if (parsed_text.hasOwnProperty('address')) { + vs.var( 'input:address', parsed_text.address ); + } + // house number if( parsed_text.hasOwnProperty('number') ){ vs.var( 'input:housenumber', parsed_text.number ); diff --git a/routes/v1.js b/routes/v1.js index 6c5afd58..a37bef90 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -8,6 +8,7 @@ var sanitizers = { place: require('../sanitizer/place'), search: require('../sanitizer/search'), search_fallback: require('../sanitizer/search_fallback'), + component: require('../sanitizer/component'), reverse: require('../sanitizer/reverse'), nearby: require('../sanitizer/nearby') }; @@ -28,13 +29,15 @@ var controllers = { var queries = { libpostal: require('../query/search'), - fallback_to_old_prod: require('../query/search_original') + fallback_to_old_prod: require('../query/search_original'), + component: require('../query/search_component') }; /** ----------------------- controllers ----------------------- **/ var postProc = { trimByGranularity: require('../middleware/trimByGranularity'), + trimByGranularityComponent: require('../middleware/trimByGranularityComponent'), distances: require('../middleware/distance'), confidenceScores: require('../middleware/confidenceScore'), confidenceScoresFallback: require('../middleware/confidenceScoreFallback'), @@ -92,6 +95,28 @@ function addRoutes(app, peliasConfig) { postProc.geocodeJSON(peliasConfig, base), postProc.sendJSON ]), + component: createRouter([ + sanitizers.component.middleware, + middleware.calcSize(), + // 2nd parameter is `backend` which gets initialized internally + // 3rd parameter is which query module to use, use fallback/geodisambiguation + // first, then use original search strategy if first query didn't return anything + controllers.search(peliasConfig, undefined, queries.component), + // sanitizers.search_fallback.middleware, + // controllers.search(peliasConfig, undefined, queries.fallback_to_old_prod), + postProc.trimByGranularityComponent(), + postProc.distances('focus.point.'), + postProc.confidenceScores(peliasConfig), + postProc.confidenceScoresFallback(), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.geocodeJSON(peliasConfig, base), + postProc.sendJSON + ]), autocomplete: createRouter([ sanitizers.autocomplete.middleware, controllers.search(peliasConfig, null, require('../query/autocomplete')), @@ -172,6 +197,7 @@ function addRoutes(app, peliasConfig) { app.get ( base + 'autocomplete', routers.autocomplete ); app.get ( base + 'search', routers.search ); app.post( base + 'search', routers.search ); + app.get ( base + 'component', routers.component ); app.get ( base + 'reverse', routers.reverse ); app.get ( base + 'nearby', routers.nearby ); diff --git a/sanitizer/component.js b/sanitizer/component.js new file mode 100644 index 00000000..0ab44fc1 --- /dev/null +++ b/sanitizer/component.js @@ -0,0 +1,26 @@ +var type_mapping = require('../helper/type_mapping'); + +var sanitizeAll = require('../sanitizer/sanitizeAll'), + sanitizers = { + quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'), + singleScalarParameters: require('../sanitizer/_single_scalar_parameters'), + synthesize_analysis: require('../sanitizer/_synthesize_analysis'), + size: require('../sanitizer/_size')(/* use defaults*/), + layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), + sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping), + // depends on the layers and sources sanitizers, must be run after them + sources_and_layers: require('../sanitizer/_sources_and_layers'), + private: require('../sanitizer/_flag_bool')('private', false), + geo_search: require('../sanitizer/_geo_search'), + boundary_country: require('../sanitizer/_boundary_country'), + categories: require('../sanitizer/_categories') + }; + +var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); }; + +// middleware +module.exports.middleware = function( req, res, next ){ + sanitize( req, function( err, clean ){ + next(); + }); +}; diff --git a/test/unit/middleware/trimByGranularityComponent.js b/test/unit/middleware/trimByGranularityComponent.js new file mode 100644 index 00000000..93bb4957 --- /dev/null +++ b/test/unit/middleware/trimByGranularityComponent.js @@ -0,0 +1,431 @@ +var trimByGranularity = require('../../../middleware/trimByGranularityComponent')(); + +module.exports.tests = {}; + +module.exports.tests.trimByGranularity = function(test, common) { + test('empty res and req should not throw exception', function(t) { + function testIt() { + trimByGranularity({}, {}, function() {}); + } + + t.doesNotThrow(testIt, 'an exception should not have been thrown'); + t.end(); + }); + + test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'venue 1', _matched_queries: ['fallback.venue'] }, + { name: 'venue 2', _matched_queries: ['fallback.venue'] }, + { name: 'street 1', _matched_queries: ['fallback.street'] }, + { name: 'address 1', _matched_queries: ['fallback.address'] }, + { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'venue 1', _matched_queries: ['fallback.venue'] }, + { name: 'venue 2', _matched_queries: ['fallback.venue'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only venue records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only streets when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'street 1', _matched_queries: ['fallback.street'] }, + { name: 'street 2', _matched_queries: ['fallback.street'] }, + { name: 'address 1', _matched_queries: ['fallback.address'] }, + { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'street 1', _matched_queries: ['fallback.street'] }, + { name: 'street 2', _matched_queries: ['fallback.street'] } + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only street records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only address when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'address 1', _matched_queries: ['fallback.address'] }, + { name: 'address 2', _matched_queries: ['fallback.address'] }, + { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'address 1', _matched_queries: ['fallback.address'] }, + { name: 'address 2', _matched_queries: ['fallback.address'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only address records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only neighbourhoods when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] }, + { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only neighbourhood records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only localities when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'locality 2', _matched_queries: ['fallback.locality'] }, + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'locality 1', _matched_queries: ['fallback.locality'] }, + { name: 'locality 2', _matched_queries: ['fallback.locality'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only locality records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only localadmins when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'localadmin 2', _matched_queries: ['fallback.localadmin'] }, + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] }, + { name: 'localadmin 2', _matched_queries: ['fallback.localadmin'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only localadmin records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only counties when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'county 2', _matched_queries: ['fallback.county'] }, + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'county 1', _matched_queries: ['fallback.county'] }, + { name: 'county 2', _matched_queries: ['fallback.county'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only county records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only macrocounties when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'macrocounty 2', _matched_queries: ['fallback.macrocounty'] }, + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] }, + { name: 'macrocounty 2', _matched_queries: ['fallback.macrocounty'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only macrocounty records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only regions when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'region 2', _matched_queries: ['fallback.region'] }, + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'region 1', _matched_queries: ['fallback.region'] }, + { name: 'region 2', _matched_queries: ['fallback.region'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only region records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only macroregions when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'macroregion 2', _matched_queries: ['fallback.macroregion'] }, + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] }, + { name: 'macroregion 2', _matched_queries: ['fallback.macroregion'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only macroregion records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only dependencies when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'dependency 2', _matched_queries: ['fallback.dependency'] }, + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'dependency 1', _matched_queries: ['fallback.dependency'] }, + { name: 'dependency 2', _matched_queries: ['fallback.dependency'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only dependency records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('all records with fallback.* matched_queries name should retain only countries when they are most granular', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'country 2', _matched_queries: ['fallback.country'] }, + { name: 'unknown', _matched_queries: ['fallback.unknown'] } + ] + }; + + var expected_data = [ + { name: 'country 1', _matched_queries: ['fallback.country'] }, + { name: 'country 2', _matched_queries: ['fallback.country'] }, + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'only country records should be here'); + t.end(); + }); + } + + testIt(); + }); + + test('presence of any non-fallback.* named queries should not trim', function(t) { + var req = { clean: {} }; + + var res = { + data: [ + { name: 'region', _matched_queries: ['fallback.region'] }, + { name: 'country', _matched_queries: ['fallback.country'] }, + { name: 'result with non-named query' } + ] + }; + + var expected_data = [ + { name: 'region', _matched_queries: ['fallback.region'] }, + { name: 'country', _matched_queries: ['fallback.country'] }, + { name: 'result with non-named query' } + ]; + + function testIt() { + trimByGranularity(req, res, function() { + t.deepEquals(res.data, expected_data, 'all should results should have been retained'); + t.end(); + }); + } + + testIt(); + + }); + +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('[middleware] trimByGranularity: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/component.js b/test/unit/query/component.js new file mode 100644 index 00000000..5fe2ee15 --- /dev/null +++ b/test/unit/query/component.js @@ -0,0 +1,283 @@ +var generate = require('../../../query/component'); +var fs = require('fs'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof generate, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.tests.query = function(test, common) { + test('valid search + focus + bbox', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', + querySize: 10, + 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, + 'boundary.rect.min_lat': 47.47, + 'boundary.rect.max_lon': -61.84, + 'boundary.rect.max_lat': 11.51, + 'boundary.rect.min_lon': -103.16, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_focus_bbox'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox'); + t.end(); + }); + + test('valid search + bbox', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', + querySize: 10, + 'boundary.rect.min_lat': 47.47, + 'boundary.rect.max_lon': -61.84, + 'boundary.rect.max_lat': 11.51, + 'boundary.rect.min_lon': -103.16, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_bbox'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_bbox'); + t.end(); + }); + + test('valid lingustic-only search', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_only'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_only'); + t.end(); + }); + + test('search search + focus', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_focus'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_focus'); + t.end(); + }); + + test('search search + viewport', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + 'focus.viewport.min_lat': 28.49136, + 'focus.viewport.max_lat': 30.49136, + 'focus.viewport.min_lon': -87.50622, + 'focus.viewport.max_lon': -77.50622, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_viewport'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_viewport'); + t.end(); + }); + + // viewport scale sizing currently disabled. + // ref: https://github.com/pelias/api/pull/388 + test('search with viewport diagonal < 1km should set scale to 1km', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + 'focus.viewport.min_lat': 28.49135, + 'focus.viewport.max_lat': 28.49137, + 'focus.viewport.min_lon': -87.50622, + 'focus.viewport.max_lon': -87.50624, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_viewport_min_diagonal'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'valid search query'); + t.end(); + }); + + test('search search + focus on null island', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + 'focus.point.lat': 0, 'focus.point.lon': 0, + layers: ['test'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_linguistic_focus_null_island'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island'); + t.end(); + }); + + test('parsed_text with all fields should use FallbackQuery', function(t) { + var clean = { + parsed_text: { + query: 'query value', + category: 'category value', + number: 'number value', + street: 'street value', + neighbourhood: 'neighbourhood value', + borough: 'borough value', + postalcode: 'postalcode value', + city: 'city value', + county: 'county value', + state: 'state value', + country: 'country value' + } + }; + + var query = generate(clean); + + var compiled = JSON.parse(JSON.stringify(query)); + var expected = require('../fixture/search_fallback'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'fallbackQuery'); + t.end(); + + }); + + test('parsed_text with single admin field should return undefined', function(t) { + ['neighbourhood', 'borough', 'city', 'county', 'state', 'country'].forEach(function(placeType) { + var clean = { + parsed_text: {} + }; + + clean.parsed_text[placeType] = placeType + ' value'; + + var query = generate(clean); + + t.equals(query, undefined, 'geodisambiguationQuery'); + + }); + + t.end(); + + }); + + test('valid boundary.country search', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + text: 'test', querySize: 10, + layers: ['test'], + 'boundary.country': 'ABC' + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_boundary_country'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search: valid boundary.country query'); + t.end(); + }); + + test('valid sources filter', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + 'text': 'test', + 'sources': ['test_source'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_with_source_filtering'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering'); + t.end(); + }); + + test('categories filter', function(t) { + var clean = { + parsed_text: { + street: 'street value' + }, + 'text': 'test', + 'categories': ['retail','food'] + }; + + var query = generate(clean); + + var compiled = JSON.parse( JSON.stringify( query ) ); + var expected = require('../fixture/search_with_category_filtering'); + + t.deepEqual(compiled.type, 'fallback', 'query type set'); + t.deepEqual(compiled.body, expected, 'valid search query with category filtering'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('search query ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/query/text_parser.js b/test/unit/query/text_parser.js index 34830c7f..839ddfbd 100644 --- a/test/unit/query/text_parser.js +++ b/test/unit/query/text_parser.js @@ -21,6 +21,7 @@ module.exports.tests.query = function(test, common) { t.false(vs.isset('input:category')); t.false(vs.isset('input:housenumber')); t.false(vs.isset('input:street')); + t.false(vs.isset('input:address')); t.false(vs.isset('input:neighbourhood')); t.false(vs.isset('input:borough')); t.false(vs.isset('input:postcode')); @@ -38,6 +39,7 @@ module.exports.tests.query = function(test, common) { category: 'category value', number: 'number value', street: 'street value', + address: 'address value', neighbourhood: 'neighbourhood value', borough: 'borough value', postalcode: 'postalcode value', @@ -54,6 +56,7 @@ module.exports.tests.query = function(test, common) { t.equals(vs.var('input:category').toString(), 'category value'); t.equals(vs.var('input:housenumber').toString(), 'number value'); t.equals(vs.var('input:street').toString(), 'street value'); + t.equals(vs.var('input:address').toString(), 'address value'); t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value'); t.equals(vs.var('input:borough').toString(), 'borough value'); t.equals(vs.var('input:postcode').toString(), 'postalcode value'); diff --git a/test/unit/run.js b/test/unit/run.js index 63c2dd4b..abcefd47 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -30,6 +30,7 @@ var tests = [ require('./middleware/sendJSON'), require('./middleware/normalizeParentIds'), require('./middleware/trimByGranularity'), + require('./middleware/trimByGranularityComponent'), require('./query/autocomplete'), require('./query/autocomplete_defaults'), require('./query/search_defaults'), @@ -58,6 +59,7 @@ var tests = [ require('./sanitizer/nearby'), require('./src/backend'), require('./sanitizer/autocomplete'), + require('./sanitizer/component'), require('./sanitizer/place'), require('./sanitizer/reverse'), require('./sanitizer/sanitizeAll'), diff --git a/test/unit/sanitizer/component.js b/test/unit/sanitizer/component.js new file mode 100644 index 00000000..7f9569dc --- /dev/null +++ b/test/unit/sanitizer/component.js @@ -0,0 +1,113 @@ +var proxyquire = require('proxyquire').noCallThru(); + +module.exports.tests = {}; + +module.exports.tests.sanitize = function(test, common) { + test('verify that all sanitizers were called as expected', function(t) { + var called_sanitizers = []; + + // rather than re-verify the functionality of all the sanitizers, this test just verifies that they + // were all called correctly + var search = proxyquire('../../../sanitizer/component', { + '../sanitizer/_deprecate_quattroshapes': function() { + called_sanitizers.push('_deprecate_quattroshapes'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_single_scalar_parameters': function() { + called_sanitizers.push('_single_scalar_parameters'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_synthesize_analysis': function() { + called_sanitizers.push('_synthesize_analysis'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_size': function() { + if (arguments.length === 0) { + return function() { + called_sanitizers.push('_size'); + return { errors: [], warnings: [] }; + }; + + } else { + throw new Error('should not have passed any parameters to _size'); + } + + }, + '../sanitizer/_targets': function(type) { + if (['layers', 'sources'].indexOf(type) !== -1) { + return function() { + called_sanitizers.push('_targets/' + type); + return { errors: [], warnings: [] }; + }; + + } + else { + throw new Error('incorrect parameters passed to _targets'); + } + + }, + '../sanitizer/_sources_and_layers': function() { + called_sanitizers.push('_sources_and_layers'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_flag_bool': function() { + if (arguments[0] === 'private' && arguments[1] === false) { + return function() { + called_sanitizers.push('_flag_bool'); + return { errors: [], warnings: [] }; + }; + + } + else { + throw new Error('incorrect parameters passed to _flag_bool'); + } + + }, + '../sanitizer/_geo_search': function() { + called_sanitizers.push('_geo_search'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_boundary_country': function() { + called_sanitizers.push('_boundary_country'); + return { errors: [], warnings: [] }; + }, + '../sanitizer/_categories': function() { + called_sanitizers.push('_categories'); + return { errors: [], warnings: [] }; + }, + }); + + var expected_sanitizers = [ + '_deprecate_quattroshapes', + '_single_scalar_parameters', + '_synthesize_analysis', + '_size', + '_targets/layers', + '_targets/sources', + '_sources_and_layers', + '_flag_bool', + '_geo_search', + '_boundary_country', + '_categories' + ]; + + var req = {}; + var res = {}; + + search.middleware(req, res, function(){ + t.deepEquals(called_sanitizers, expected_sanitizers); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('SANTIZE /component ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +};