Browse Source

Merge pull request #755 from pelias/master

Merge master into staging
pull/756/head
Julian Simioni 8 years ago committed by GitHub
parent
commit
2214eef4bb
  1. 2
      controller/place.js
  2. 2
      controller/search.js
  3. 16
      middleware/500.js
  4. 2
      middleware/confidenceScoreFallback.js
  5. 9
      middleware/sendJSON.js
  6. 4
      middleware/trimByGranularityStructured.js
  7. 6
      package.json
  8. 22
      query/structured_geocoding.js
  9. 2
      query/text_parser_addressit.js
  10. 16
      routes/v1.js
  11. 2
      sanitizer/_single_scalar_parameters.js
  12. 42
      sanitizer/_synthesize_analysis.js
  13. 2
      sanitizer/reverse.js
  14. 2
      sanitizer/search.js
  15. 2
      sanitizer/search_fallback.js
  16. 2
      sanitizer/structured_geocoding.js
  17. 13
      service/mget.js
  18. 15
      service/search.js
  19. 30
      test/ciao/autocomplete/non_scalar_parameter_array.coffee
  20. 30
      test/ciao/autocomplete/non_scalar_parameter_object.coffee
  21. 30
      test/ciao/reverse/non_scalar_parameter_array.coffee
  22. 2
      test/ciao/reverse/non_scalar_parameter_object.coffee
  23. 2
      test/ciao/search/address_parsing.coffee
  24. 30
      test/ciao/search/non_scalar_parameter_array.coffee
  25. 30
      test/ciao/search/non_scalar_parameter_object.coffee
  26. 0
      test/unit/fixture/structured_geocoding/boundary_country.json
  27. 191
      test/unit/fixture/structured_geocoding/fallback.json
  28. 0
      test/unit/fixture/structured_geocoding/linguistic_bbox.json
  29. 0
      test/unit/fixture/structured_geocoding/linguistic_focus.json
  30. 0
      test/unit/fixture/structured_geocoding/linguistic_focus_bbox.json
  31. 0
      test/unit/fixture/structured_geocoding/linguistic_focus_null_island.json
  32. 0
      test/unit/fixture/structured_geocoding/linguistic_only.json
  33. 0
      test/unit/fixture/structured_geocoding/linguistic_viewport.json
  34. 0
      test/unit/fixture/structured_geocoding/linguistic_viewport_min_diagonal.json
  35. 0
      test/unit/fixture/structured_geocoding/with_source_filtering.json
  36. 26
      test/unit/middleware/trimByGranularityStructured.js
  37. 33
      test/unit/query/structured_geocoding.js
  38. 6
      test/unit/run.js
  39. 9
      test/unit/sanitizer/_single_scalar_parameters.js
  40. 124
      test/unit/sanitizer/_synthesize_analysis.js
  41. 2
      test/unit/sanitizer/nearby.js
  42. 2
      test/unit/sanitizer/reverse.js
  43. 2
      test/unit/sanitizer/search.js
  44. 6
      test/unit/sanitizer/structured_geocoding.js

2
controller/place.js

@ -1,5 +1,5 @@
var service = { mget: require('../service/mget') };
var logger = require('pelias-logger').get('api:controller:place');
var logger = require('pelias-logger').get('api');
function setup( config, backend ){

2
controller/search.js

@ -1,7 +1,7 @@
var _ = require('lodash');
var service = { search: require('../service/search') };
var logger = require('pelias-logger').get('api:controller:search');
var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' );
function setup( config, backend, query ){

16
middleware/500.js

@ -1,13 +1,19 @@
var logger = require( 'pelias-logger' ).get( 'middleware-500' );
var check = require('check-types'),
logger = require( 'pelias-logger' ).get( 'api' );
// handle application errors
function middleware(err, req, res, next) {
logger.error( 'Error: `%s`. Stack trace: `%s`.', err, err.stack );
res.header('Cache-Control','public');
var error = (err && err.message) ? err.message : err;
if( res.statusCode < 400 ){ res.status(500); }
res.json({ error: typeof error === 'string' ? error : 'internal server error' });
if( res.statusCode < 400 ){
logger.info( 'status code changed from', res.statusCode, 'to 500' );
res.status(500);
}
var error = ( err && err.message ) ? err.message : err;
res.header('Cache-Control','public');
res.json({ error: check.nonEmptyString( error ) ? error : 'internal server error' });
}
module.exports = middleware;

2
middleware/confidenceScoreFallback.js

@ -10,7 +10,7 @@
*/
var check = require('check-types');
var logger = require('pelias-logger').get('api-confidence');
var logger = require('pelias-logger').get('api');
function setup() {
return computeScores;

9
middleware/sendJSON.js

@ -1,5 +1,6 @@
var check = require('check-types'),
es = require('elasticsearch'),
logger = require( 'pelias-logger' ).get( 'api' ),
exceptions = require('elasticsearch-exceptions/lib/exceptions/SupportedExceptions');
// create a list of regular expressions to match against.
@ -42,7 +43,10 @@ function sendJSONResponse(req, res, next) {
if( err instanceof es.errors.RequestTimeout ){ statusCode = Math.max( statusCode, 408 ); }
else if( err instanceof es.errors.NoConnections ){ statusCode = Math.max( statusCode, 502 ); }
else if( err instanceof es.errors.ConnectionFault ){ statusCode = Math.max( statusCode, 502 ); }
else { statusCode = Math.max( statusCode, 500 ); }
else {
logger.warn( 'unknown geocoding error object:', err.constructor.name, err );
statusCode = Math.max( statusCode, 500 );
}
/*
some elasticsearch errors are only returned as strings (not instances of Error).
@ -55,7 +59,10 @@ function sendJSONResponse(req, res, next) {
statusCode = Math.max( statusCode, 500 );
break; // break on first match
}
logger.warn( 'unknown geocoding error string:', err );
}
} else {
logger.warn( 'unknown geocoding error type:', typeof err, err );
}
});
}

4
middleware/trimByGranularityComponent.js → middleware/trimByGranularityStructured.js

@ -22,8 +22,8 @@ const _ = require('lodash');
// supplied we want to retain borough=Manhattan and city=Manhattan results
const layers = [
'venue',
'street',
'address',
'street',
'neighbourhood',
['borough', 'locality'],
'localadmin',
@ -44,8 +44,8 @@ const layers = [
// city=Manhattan
const explicit_borough_layers = [
'venue',
'street',
'address',
'street',
'neighbourhood',
'borough',
'locality',

6
package.json

@ -52,14 +52,14 @@
"markdown": "0.5.0",
"morgan": "1.7.0",
"pelias-categories": "1.1.0",
"pelias-config": "2.3.1",
"pelias-config": "2.4.0",
"pelias-labels": "1.5.1",
"pelias-logger": "0.1.0",
"pelias-model": "4.3.0",
"pelias-query": "8.9.0",
"pelias-query": "8.11.0",
"pelias-text-analyzer": "1.6.0",
"stats-lite": "2.0.3",
"through2": "2.0.1"
"through2": "^2.0.3"
},
"devDependencies": {
"ciao": "^0.6.0",

22
query/component_geocoding.js → query/structured_geocoding.js

@ -6,21 +6,21 @@ var peliasQuery = require('pelias-query'),
//------------------------------
// general-purpose search query
//------------------------------
var componentQuery = new peliasQuery.layout.ComponentFallbackQuery();
var structuredQuery = new peliasQuery.layout.StructuredFallbackQuery();
// scoring boost
componentQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
componentQuery.score( peliasQuery.view.popularity_only_function );
componentQuery.score( peliasQuery.view.population_only_function );
structuredQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
structuredQuery.score( peliasQuery.view.popularity_only_function );
structuredQuery.score( peliasQuery.view.population_only_function );
// --------------------------------
// non-scoring hard filters
componentQuery.filter( peliasQuery.view.boundary_country );
componentQuery.filter( peliasQuery.view.boundary_circle );
componentQuery.filter( peliasQuery.view.boundary_rect );
componentQuery.filter( peliasQuery.view.sources );
componentQuery.filter( peliasQuery.view.layers );
componentQuery.filter( peliasQuery.view.categories );
structuredQuery.filter( peliasQuery.view.boundary_country );
structuredQuery.filter( peliasQuery.view.boundary_circle );
structuredQuery.filter( peliasQuery.view.boundary_rect );
structuredQuery.filter( peliasQuery.view.sources );
structuredQuery.filter( peliasQuery.view.layers );
structuredQuery.filter( peliasQuery.view.categories );
// --------------------------------
/**
@ -105,7 +105,7 @@ function generateQuery( clean ){
function getQuery(vs) {
return {
type: 'fallback',
body: componentQuery.render(vs)
body: structuredQuery.render(vs)
};
}

2
query/text_parser_addressit.js

@ -32,8 +32,6 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
// ?
else {
logger.warn( 'chaos monkey asks: what happens now?' );
logger.warn( parsed_text );
try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
}
// ==== add parsed matches [address components] ====

16
routes/v1.js

@ -8,7 +8,7 @@ var sanitizers = {
place: require('../sanitizer/place'),
search: require('../sanitizer/search'),
search_fallback: require('../sanitizer/search_fallback'),
component_geocoding: require('../sanitizer/component_geocoding'),
structured_geocoding: require('../sanitizer/structured_geocoding'),
reverse: require('../sanitizer/reverse'),
nearby: require('../sanitizer/nearby')
};
@ -30,14 +30,14 @@ var controllers = {
var queries = {
libpostal: require('../query/search'),
fallback_to_old_prod: require('../query/search_original'),
component_geocoding: require('../query/component_geocoding')
structured_geocoding: require('../query/structured_geocoding')
};
/** ----------------------- controllers ----------------------- **/
var postProc = {
trimByGranularity: require('../middleware/trimByGranularity'),
trimByGranularityComponent: require('../middleware/trimByGranularityComponent'),
trimByGranularityStructured: require('../middleware/trimByGranularityStructured'),
distances: require('../middleware/distance'),
confidenceScores: require('../middleware/confidenceScore'),
confidenceScoresFallback: require('../middleware/confidenceScoreFallback'),
@ -95,11 +95,11 @@ function addRoutes(app, peliasConfig) {
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
component: createRouter([
sanitizers.component_geocoding.middleware,
structured: createRouter([
sanitizers.structured_geocoding.middleware,
middleware.calcSize(),
controllers.search(peliasConfig, undefined, queries.component_geocoding),
postProc.trimByGranularityComponent(),
controllers.search(peliasConfig, undefined, queries.structured_geocoding),
postProc.trimByGranularityStructured(),
postProc.distances('focus.point.'),
postProc.confidenceScores(peliasConfig),
postProc.confidenceScoresFallback(),
@ -193,7 +193,7 @@ function addRoutes(app, peliasConfig) {
app.get ( base + 'autocomplete', routers.autocomplete );
app.get ( base + 'search', routers.search );
app.post( base + 'search', routers.search );
app.get ( base + 'beta/component', routers.component );
app.get ( base + 'search/structured', routers.structured );
app.get ( base + 'reverse', routers.reverse );
app.get ( base + 'nearby', routers.nearby );

2
sanitizer/_single_scalar_parameters.js

@ -10,8 +10,10 @@ function sanitize( raw, clean ){
Object.keys(raw).forEach(function(key) {
if (_.isArray(raw[key])) {
messages.errors.push('\'' + key + '\' parameter can only have one value');
delete raw[key];
} else if (_.isObject(raw[key])) {
messages.errors.push('\'' + key + '\' parameter must be a scalar');
delete raw[key];
}
});

42
sanitizer/_synthesize_analysis.js

@ -1,4 +1,5 @@
const _ = require('lodash');
const text_analyzer = require('pelias-text-analyzer');
const fields = {
'address': 'address',
@ -20,6 +21,19 @@ function isPostalCodeOnly(parsed_text) {
parsed_text.hasOwnProperty('postalcode');
}
// figure out which field contains the probable house number, prefer number
// libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street
// so because we're treating the entire field as a street address, it's safe
// to assume that an identified postcode is actually a house number.
function getHouseNumberField(analyzed_address) {
for (var field of ['number', 'postalcode']) {
if (analyzed_address.hasOwnProperty(field)) {
return field;
}
}
}
function sanitize( raw, clean ){
// error & warning messages
@ -43,6 +57,34 @@ function sanitize( raw, clean ){
`at least one of the following fields is required: ${Object.keys(fields).join(', ')}`);
}
if (clean.parsed_text.hasOwnProperty('address')) {
const analyzed_address = text_analyzer.parse(clean.parsed_text.address);
const house_number_field = getHouseNumberField(analyzed_address);
// if we're fairly certain that libpostal identified a house number
// (from either the house_number or postcode field), place it into the
// number field and remove the first instance of that value from address
// and assign to street
// eg - '1090 N Charlotte St' becomes number=1090 and street=N Charlotte St
if (house_number_field) {
clean.parsed_text.number = analyzed_address[house_number_field];
// remove the first instance of the number and trim whitespace
clean.parsed_text.street = _.trim(_.replace(clean.parsed_text.address, clean.parsed_text.number, ''));
} else {
// otherwise no house number was identifiable, so treat the entire input
// as a street
clean.parsed_text.street = clean.parsed_text.address;
}
// the address field no longer means anything since it's been parsed, so remove it
delete clean.parsed_text.address;
}
return messages;
}

2
sanitizer/reverse.js

@ -2,8 +2,8 @@
var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
// depends on the layers and sources sanitizers, must be run after them

2
sanitizer/search.js

@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
text: require('../sanitizer/_text'),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
size: require('../sanitizer/_size')(/* use defaults*/),

2
sanitizer/search_fallback.js

@ -4,7 +4,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'),
};
var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
var logger = require('pelias-logger').get('api:controller:search_fallback');
var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' );
// middleware

2
sanitizer/component_geocoding.js → sanitizer/structured_geocoding.js

@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
synthesize_analysis: require('../sanitizer/_synthesize_analysis'),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
size: require('../sanitizer/_size')(/* use defaults*/),

13
service/mget.js

@ -11,7 +11,7 @@
**/
var peliasLogger = require( 'pelias-logger' ).get( 'service/mget' );
var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, query, cb ){
@ -24,8 +24,17 @@ function service( backend, query, cb ){
// query new backend
backend().client.mget( cmd, function( err, data ){
// log total ms elasticsearch reported the query took to execute
if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
}
// handle backend errors
if( err ){ return cb( err ); }
if( err ){
logger.error( 'backend error', err );
return cb( err );
}
// map returned documents
var docs = [];

15
service/search.js

@ -5,18 +5,23 @@
**/
var peliasLogger = require( 'pelias-logger' ).get( 'service/search' );
var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, cmd, cb ){
// query new backend
backend().client.search( cmd, function( err, data ){
// handle backend errors
if( err ){ return cb( err ); }
// log total ms elasticsearch reported the query took to execute
peliasLogger.verbose( 'time elasticsearch reported:', data.took / 1000 );
if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
}
// handle backend errors
if( err ){
logger.error( 'backend error', err );
return cb( err );
}
// map returned documents
var docs = [];

30
test/ciao/autocomplete/non_scalar_parameter_array.coffee

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/autocomplete?text=A&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

30
test/ciao/autocomplete/non_scalar_parameter_object.coffee

@ -0,0 +1,30 @@
#> define parameter as object
path: '/v1/autocomplete?text=A&parameter[idx]=value'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

30
test/ciao/reverse/non_scalar_parameter_array.coffee

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/reverse?point.lat=1&point.lon=1&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

2
test/ciao/reverse/non_scalar_parameter.coffee → test/ciao/reverse/non_scalar_parameter_object.coffee

@ -1,5 +1,5 @@
#> set size
#> define parameter as object
path: '/v1/reverse?point.lat=1&point.lon=1&parameter[idx]=value'
#? 200 ok

2
test/ciao/search/address_parsing.coffee

@ -35,7 +35,7 @@ json.geocoding.query['size'].should.eql 10
#? address parsing
json.geocoding.query.parsed_text['number'].should.eql '30'
json.geocoding.query.parsed_text['street'].should.eql 'w 26th st'
json.geocoding.query.parsed_text['state'].should.eql 'ny'
json.geocoding.query.parsed_text['state'].should.eql 'NY'
json.features[0].properties.confidence.should.eql 1
json.features[0].properties.match_type.should.eql "exact"

30
test/ciao/search/non_scalar_parameter_array.coffee

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/search?text=A&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

30
test/ciao/search/non_scalar_parameter_object.coffee

@ -0,0 +1,30 @@
#> define parameter as object
path: '/v1/search?text=A&parameter[idx]=value'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

0
test/unit/fixture/component_geocoding/boundary_country.json → test/unit/fixture/structured_geocoding/boundary_country.json

191
test/unit/fixture/component_geocoding/fallback.json → test/unit/fixture/structured_geocoding/fallback.json

@ -6,6 +6,197 @@
"query": {
"bool": {
"should": [
{
"bool": {
"_name": "fallback.address",
"must": [
{
"match_phrase": {
"address_parts.number": "number value"
}
},
{
"match_phrase": {
"address_parts.street": "street value"
}
},
{
"multi_match": {
"query": "neighbourhood value",
"type": "phrase",
"fields": [
"parent.neighbourhood",
"parent.neighbourhood_a"
]
}
},
{
"multi_match": {
"query": "borough value",
"type": "phrase",
"fields": [
"parent.borough",
"parent.borough_a"
]
}
},
{
"multi_match": {
"query": "city value",
"type": "phrase",
"fields": [
"parent.locality",
"parent.locality_a",
"parent.localadmin",
"parent.localadmin_a"
]
}
},
{
"multi_match": {
"query": "county value",
"type": "phrase",
"fields": [
"parent.county",
"parent.county_a",
"parent.macrocounty",
"parent.macrocounty_a"
]
}
},
{
"multi_match": {
"query": "state value",
"type": "phrase",
"fields": [
"parent.region",
"parent.region_a",
"parent.macroregion",
"parent.macroregion_a"
]
}
},
{
"multi_match": {
"query": "country value",
"type": "phrase",
"fields": [
"parent.country",
"parent.country_a",
"parent.dependency",
"parent.dependency_a"
]
}
}
],
"should": [
{
"match_phrase": {
"address_parts.zip": "postalcode value"
}
}
],
"filter": {
"term": {
"layer": "address"
}
},
"boost": 10
}
},
{
"bool": {
"_name": "fallback.street",
"must": [
{
"match_phrase": {
"address_parts.street": "street value"
}
},
{
"multi_match": {
"query": "neighbourhood value",
"type": "phrase",
"fields": [
"parent.neighbourhood",
"parent.neighbourhood_a"
]
}
},
{
"multi_match": {
"query": "borough value",
"type": "phrase",
"fields": [
"parent.borough",
"parent.borough_a"
]
}
},
{
"multi_match": {
"query": "city value",
"type": "phrase",
"fields": [
"parent.locality",
"parent.locality_a",
"parent.localadmin",
"parent.localadmin_a"
]
}
},
{
"multi_match": {
"query": "county value",
"type": "phrase",
"fields": [
"parent.county",
"parent.county_a",
"parent.macrocounty",
"parent.macrocounty_a"
]
}
},
{
"multi_match": {
"query": "state value",
"type": "phrase",
"fields": [
"parent.region",
"parent.region_a",
"parent.macroregion",
"parent.macroregion_a"
]
}
},
{
"multi_match": {
"query": "country value",
"type": "phrase",
"fields": [
"parent.country",
"parent.country_a",
"parent.dependency",
"parent.dependency_a"
]
}
}
],
"should": [
{
"match_phrase": {
"address_parts.zip": "postalcode value"
}
}
],
"filter": {
"term": {
"layer": "street"
}
},
"boost": 5
}
},
{
"bool": {
"_name": "fallback.neighbourhood",

0
test/unit/fixture/component_geocoding/linguistic_bbox.json → test/unit/fixture/structured_geocoding/linguistic_bbox.json

0
test/unit/fixture/component_geocoding/linguistic_focus.json → test/unit/fixture/structured_geocoding/linguistic_focus.json

0
test/unit/fixture/component_geocoding/linguistic_focus_bbox.json → test/unit/fixture/structured_geocoding/linguistic_focus_bbox.json

0
test/unit/fixture/component_geocoding/linguistic_focus_null_island.json → test/unit/fixture/structured_geocoding/linguistic_focus_null_island.json

0
test/unit/fixture/component_geocoding/linguistic_only.json → test/unit/fixture/structured_geocoding/linguistic_only.json

0
test/unit/fixture/component_geocoding/linguistic_viewport.json → test/unit/fixture/structured_geocoding/linguistic_viewport.json

0
test/unit/fixture/component_geocoding/linguistic_viewport_min_diagonal.json → test/unit/fixture/structured_geocoding/linguistic_viewport_min_diagonal.json

0
test/unit/fixture/component_geocoding/with_source_filtering.json → test/unit/fixture/structured_geocoding/with_source_filtering.json

26
test/unit/middleware/trimByGranularityComponent.js → test/unit/middleware/trimByGranularityStructured.js

@ -1,4 +1,4 @@
var trimByGranularity = require('../../../middleware/trimByGranularityComponent')();
var trimByGranularity = require('../../../middleware/trimByGranularityStructured')();
module.exports.tests = {};
@ -19,8 +19,8 @@ module.exports.tests.trimByGranularity = function(test, common) {
data: [
{ name: 'venue 1', _matched_queries: ['fallback.venue'] },
{ name: 'venue 2', _matched_queries: ['fallback.venue'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -55,9 +55,9 @@ module.exports.tests.trimByGranularity = function(test, common) {
var res = {
data: [
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -73,13 +73,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
};
var expected_data = [
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] }
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] }
];
function testIt() {
trimByGranularity(req, res, function() {
t.deepEquals(res.data, expected_data, 'only street records should be here');
t.deepEquals(res.data, expected_data, 'only address records should be here');
t.end();
});
}
@ -87,13 +87,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
testIt();
});
test('all records with fallback.* matched_queries name should retain only address when they are most granular', function(t) {
test('all records with fallback.* matched_queries name should retain only street when they are most granular', function(t) {
var req = { clean: {} };
var res = {
data: [
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -109,13 +109,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
};
var expected_data = [
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
];
function testIt() {
trimByGranularity(req, res, function() {
t.deepEquals(res.data, expected_data, 'only address records should be here');
t.deepEquals(res.data, expected_data, 'only street records should be here');
t.end();
});
}

33
test/unit/query/component_geocoding.js → test/unit/query/structured_geocoding.js

@ -1,4 +1,4 @@
var generate = require('../../../query/component_geocoding');
var generate = require('../../../query/structured_geocoding');
var fs = require('fs');
module.exports.tests = {};
@ -14,7 +14,6 @@ module.exports.tests.query = function(test, common) {
test('valid search + focus + bbox', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test',
querySize: 10,
@ -29,7 +28,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus_bbox');
var expected = require('../fixture/structured_geocoding/linguistic_focus_bbox');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox');
@ -39,7 +38,6 @@ module.exports.tests.query = function(test, common) {
test('valid search + bbox', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test',
querySize: 10,
@ -53,7 +51,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_bbox');
var expected = require('../fixture/structured_geocoding/linguistic_bbox');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_bbox');
@ -63,7 +61,6 @@ module.exports.tests.query = function(test, common) {
test('valid lingustic-only search', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
layers: ['test']
@ -72,7 +69,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_only');
var expected = require('../fixture/structured_geocoding/linguistic_only');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_only');
@ -82,7 +79,6 @@ module.exports.tests.query = function(test, common) {
test('search search + focus', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
@ -92,7 +88,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus');
var expected = require('../fixture/structured_geocoding/linguistic_focus');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus');
@ -102,7 +98,6 @@ module.exports.tests.query = function(test, common) {
test('search search + viewport', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.viewport.min_lat': 28.49136,
@ -115,7 +110,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_viewport');
var expected = require('../fixture/structured_geocoding/linguistic_viewport');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_viewport');
@ -127,7 +122,6 @@ module.exports.tests.query = function(test, common) {
test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.viewport.min_lat': 28.49135,
@ -140,7 +134,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_viewport_min_diagonal');
var expected = require('../fixture/structured_geocoding/linguistic_viewport_min_diagonal');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'valid search query');
@ -150,7 +144,6 @@ module.exports.tests.query = function(test, common) {
test('search search + focus on null island', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.point.lat': 0, 'focus.point.lon': 0,
@ -160,7 +153,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus_null_island');
var expected = require('../fixture/structured_geocoding/linguistic_focus_null_island');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island');
@ -187,7 +180,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse(JSON.stringify(query));
var expected = require('../fixture/component_geocoding/fallback');
var expected = require('../fixture/structured_geocoding/fallback');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'fallbackQuery');
@ -198,7 +191,6 @@ module.exports.tests.query = function(test, common) {
test('valid boundary.country search', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
layers: ['test'],
@ -208,7 +200,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/boundary_country');
var expected = require('../fixture/structured_geocoding/boundary_country');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search: valid boundary.country query');
@ -218,7 +210,6 @@ module.exports.tests.query = function(test, common) {
test('valid sources filter', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
'text': 'test',
'sources': ['test_source']
@ -227,7 +218,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/with_source_filtering');
var expected = require('../fixture/structured_geocoding/with_source_filtering');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering');
@ -238,7 +229,7 @@ module.exports.tests.query = function(test, common) {
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('component_geocoding query ' + name, testFunction);
return tape('structured_geocoding query ' + name, testFunction);
}
for( var testCase in module.exports.tests ){

6
test/unit/run.js

@ -30,15 +30,15 @@ var tests = [
require('./middleware/sendJSON'),
require('./middleware/normalizeParentIds'),
require('./middleware/trimByGranularity'),
require('./middleware/trimByGranularityComponent'),
require('./middleware/trimByGranularityStructured'),
require('./query/autocomplete'),
require('./query/autocomplete_defaults'),
require('./query/component_geocoding'),
require('./query/search_defaults'),
require('./query/reverse_defaults'),
require('./query/reverse'),
require('./query/search'),
require('./query/search_original'),
require('./query/structured_geocoding'),
require('./query/text_parser'),
require('./sanitizer/_boundary_country'),
require('./sanitizer/_flag_bool'),
@ -61,7 +61,7 @@ var tests = [
require('./sanitizer/nearby'),
require('./src/backend'),
require('./sanitizer/autocomplete'),
require('./sanitizer/component_geocoding'),
require('./sanitizer/structured_geocoding'),
require('./sanitizer/place'),
require('./sanitizer/reverse'),
require('./sanitizer/sanitizeAll'),

9
test/unit/sanitizer/_single_scalar_parameters.js

@ -18,6 +18,11 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
],
warnings: []
});
// erroneous params should be deleted to avoid middleware errors
t.false(raw.arrayParameter1);
t.false(raw.arrayParameter2);
t.end();
});
@ -36,6 +41,10 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
],
warnings: []
});
// erroneous params should be deleted to avoid middleware errors
t.false(raw.objectParameter1);
t.false(raw.objectParameter2);
t.end();
});

124
test/unit/sanitizer/_synthesize_analysis.js

@ -1,13 +1,18 @@
const sanitizer = require('../../../sanitizer/_synthesize_analysis');
const _ = require('lodash');
const proxyquire = require('proxyquire').noCallThru();
module.exports.tests = {};
module.exports.tests.text_parser = function(test, common) {
test('all variables should be parsed', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {
query: ' \t query \t value \t ',
address: ' \t address \t value \t ',
neighbourhood: ' \t neighbourhood \t value \t ',
borough: ' \t borough \t value \t ',
locality: ' \t locality \t value \t ',
@ -21,7 +26,6 @@ module.exports.tests.text_parser = function(test, common) {
const expected_clean = {
parsed_text: {
address: 'address value',
neighbourhood: 'neighbourhood value',
borough: 'borough value',
city: 'locality value',
@ -42,6 +46,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('non-string and blank string values should be treated as not supplied', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
// helper to return a random value that's considered invalid
function getInvalidValue() {
return _.sample([{}, [], false, '', ' \t ', 17, undefined]);
@ -75,6 +85,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('no supplied fields should return error', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {};
const clean = {};
@ -92,6 +108,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('postalcode-only parsed_text should return error', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {
postalcode: 'postalcode value'
};
@ -113,6 +135,102 @@ module.exports.tests.text_parser = function(test, common) {
});
test('text_analyzer identifying house number should extract it and street', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Number Value Street Value Number Value');
return {
number: 'Number Value'
};
}
}});
const raw = {
address: 'Number Value Street Value Number Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
number: 'Number Value',
street: 'Street Value Number Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
test('text_analyzer identifying postalcode but not house number should assign to number and remove from address', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Number Value Street Value Number Value');
return {
postalcode: 'Number Value'
};
}
}});
const raw = {
address: 'Number Value Street Value Number Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
number: 'Number Value',
street: 'Street Value Number Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
test('text_analyzer not revealing possible number should move address to street', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Street Value');
return {};
}
}});
const raw = {
address: 'Street Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
street: 'Street Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
};
module.exports.all = function (tape, common) {

2
test/unit/sanitizer/nearby.js

@ -30,7 +30,7 @@ module.exports.tests.interface = function(test, common) {
module.exports.tests.sanitizers = function(test, common) {
test('check sanitizer list', function (t) {
var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country', 'categories'];
t.deepEqual(Object.keys(nearby.sanitizer_list), expected);
t.end();

2
test/unit/sanitizer/reverse.js

@ -36,7 +36,7 @@ module.exports.tests.interface = function(test, common) {
module.exports.tests.sanitizers = function(test, common) {
test('check sanitizer list', function (t) {
var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country'];
t.deepEqual(Object.keys(reverse.sanitizer_list), expected);
t.end();

2
test/unit/sanitizer/search.js

@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
});
var expected_sanitizers = [
'_deprecate_quattroshapes',
'_single_scalar_parameters',
'_deprecate_quattroshapes',
'_text',
'_iso2_to_iso3',
'_size',

6
test/unit/sanitizer/component_geocoding.js → test/unit/sanitizer/structured_geocoding.js

@ -8,7 +8,7 @@ module.exports.tests.sanitize = function(test, common) {
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/component_geocoding', {
var search = proxyquire('../../../sanitizer/structured_geocoding', {
'../sanitizer/_deprecate_quattroshapes': function() {
called_sanitizers.push('_deprecate_quattroshapes');
return { errors: [], warnings: [] };
@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
});
var expected_sanitizers = [
'_deprecate_quattroshapes',
'_single_scalar_parameters',
'_deprecate_quattroshapes',
'_synthesize_analysis',
'_iso2_to_iso3',
'_size',
@ -109,7 +109,7 @@ module.exports.tests.sanitize = function(test, common) {
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANTIZE /component ' + name, testFunction);
return tape('SANTIZE /structured ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
Loading…
Cancel
Save