pelias
/
api
mirror of https://github.com/pelias/api.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

Merge pull request #755 from pelias/master 

Merge master into staging
pull/756/head
Julian Simioni 8 years ago committed by GitHub
parent
aacb7afb3e 0373dccfb8
commit
2214eef4bb
44 changed files with 633 additions and 105 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      controller/place.js
  2. 2
      controller/search.js
  3. 16
      middleware/500.js
  4. 2
      middleware/confidenceScoreFallback.js
  5. 9
      middleware/sendJSON.js
  6. 4
      middleware/trimByGranularityStructured.js
  7. 6
      package.json
  8. 22
      query/structured_geocoding.js
  9. 2
      query/text_parser_addressit.js
  10. 16
      routes/v1.js
  11. 2
      sanitizer/_single_scalar_parameters.js
  12. 42
      sanitizer/_synthesize_analysis.js
  13. 2
      sanitizer/reverse.js
  14. 2
      sanitizer/search.js
  15. 2
      sanitizer/search_fallback.js
  16. 2
      sanitizer/structured_geocoding.js
  17. 13
      service/mget.js
  18. 15
      service/search.js
  19. 30
      test/ciao/autocomplete/non_scalar_parameter_array.coffee
  20. 30
      test/ciao/autocomplete/non_scalar_parameter_object.coffee
  21. 30
      test/ciao/reverse/non_scalar_parameter_array.coffee
  22. 2
      test/ciao/reverse/non_scalar_parameter_object.coffee
  23. 2
      test/ciao/search/address_parsing.coffee
  24. 30
      test/ciao/search/non_scalar_parameter_array.coffee
  25. 30
      test/ciao/search/non_scalar_parameter_object.coffee
  26. 0
      test/unit/fixture/structured_geocoding/boundary_country.json
  27. 191
      test/unit/fixture/structured_geocoding/fallback.json
  28. 0
      test/unit/fixture/structured_geocoding/linguistic_bbox.json
  29. 0
      test/unit/fixture/structured_geocoding/linguistic_focus.json
  30. 0
      test/unit/fixture/structured_geocoding/linguistic_focus_bbox.json
  31. 0
      test/unit/fixture/structured_geocoding/linguistic_focus_null_island.json
  32. 0
      test/unit/fixture/structured_geocoding/linguistic_only.json
  33. 0
      test/unit/fixture/structured_geocoding/linguistic_viewport.json
  34. 0
      test/unit/fixture/structured_geocoding/linguistic_viewport_min_diagonal.json
  35. 0
      test/unit/fixture/structured_geocoding/with_source_filtering.json
  36. 26
      test/unit/middleware/trimByGranularityStructured.js
  37. 33
      test/unit/query/structured_geocoding.js
  38. 6
      test/unit/run.js
  39. 9
      test/unit/sanitizer/_single_scalar_parameters.js
  40. 124
      test/unit/sanitizer/_synthesize_analysis.js
  41. 2
      test/unit/sanitizer/nearby.js
  42. 2
      test/unit/sanitizer/reverse.js
  43. 2
      test/unit/sanitizer/search.js
  44. 6
      test/unit/sanitizer/structured_geocoding.js

2
controller/place.js
Unescape View File

@ -1,5 +1,5 @@
var service = { mget: require('../service/mget') };
var logger = require('pelias-logger').get('api:controller:place');
var logger = require('pelias-logger').get('api');
function setup( config, backend ){

2
controller/search.js
Unescape View File

@ -1,7 +1,7 @@
var _ = require('lodash');
var service = { search: require('../service/search') };
var logger = require('pelias-logger').get('api:controller:search');
var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' );
function setup( config, backend, query ){

16
middleware/500.js
Unescape View File

@ -1,13 +1,19 @@
var logger = require( 'pelias-logger' ).get( 'middleware-500' );
var check = require('check-types'),
logger = require( 'pelias-logger' ).get( 'api' );
// handle application errors
function middleware(err, req, res, next) {
logger.error( 'Error: `%s`. Stack trace: `%s`.', err, err.stack );
res.header('Cache-Control','public');
var error = (err && err.message) ? err.message : err;
if( res.statusCode < 400 ){ res.status(500); }
res.json({ error: typeof error === 'string' ? error : 'internal server error' });
if( res.statusCode < 400 ){
logger.info( 'status code changed from', res.statusCode, 'to 500' );
res.status(500);
}
var error = ( err && err.message ) ? err.message : err;
res.header('Cache-Control','public');
res.json({ error: check.nonEmptyString( error ) ? error : 'internal server error' });
}
module.exports = middleware;

2
middleware/confidenceScoreFallback.js
Unescape View File

@ -10,7 +10,7 @@
*/
var check = require('check-types');
var logger = require('pelias-logger').get('api-confidence');
var logger = require('pelias-logger').get('api');
function setup() {
return computeScores;

9
middleware/sendJSON.js
Unescape View File

@ -1,5 +1,6 @@
var check = require('check-types'),
es = require('elasticsearch'),
logger = require( 'pelias-logger' ).get( 'api' ),
exceptions = require('elasticsearch-exceptions/lib/exceptions/SupportedExceptions');
// create a list of regular expressions to match against.
@ -42,7 +43,10 @@ function sendJSONResponse(req, res, next) {
if( err instanceof es.errors.RequestTimeout ){ statusCode = Math.max( statusCode, 408 ); }
else if( err instanceof es.errors.NoConnections ){ statusCode = Math.max( statusCode, 502 ); }
else if( err instanceof es.errors.ConnectionFault ){ statusCode = Math.max( statusCode, 502 ); }
else { statusCode = Math.max( statusCode, 500 ); }
else {
logger.warn( 'unknown geocoding error object:', err.constructor.name, err );
statusCode = Math.max( statusCode, 500 );
}
/*
some elasticsearch errors are only returned as strings (not instances of Error).
@ -55,7 +59,10 @@ function sendJSONResponse(req, res, next) {
statusCode = Math.max( statusCode, 500 );
break; // break on first match
}
logger.warn( 'unknown geocoding error string:', err );
}
} else {
logger.warn( 'unknown geocoding error type:', typeof err, err );
}
});
}

4
middleware/trimByGranularityComponent.js → middleware/trimByGranularityStructured.js
Unescape View File

@ -22,8 +22,8 @@ const _ = require('lodash');
// supplied we want to retain borough=Manhattan and city=Manhattan results
const layers = [
'venue',
'street',
'address',
'street',
'neighbourhood',
['borough', 'locality'],
'localadmin',
@ -44,8 +44,8 @@ const layers = [
// city=Manhattan
const explicit_borough_layers = [
'venue',
'street',
'address',
'street',
'neighbourhood',
'borough',
'locality',

6
package.json
Unescape View File

@ -52,14 +52,14 @@
"markdown": "0.5.0",
"morgan": "1.7.0",
"pelias-categories": "1.1.0",
"pelias-config": "2.3.1",
"pelias-config": "2.4.0",
"pelias-labels": "1.5.1",
"pelias-logger": "0.1.0",
"pelias-model": "4.3.0",
"pelias-query": "8.9.0",
"pelias-query": "8.11.0",
"pelias-text-analyzer": "1.6.0",
"stats-lite": "2.0.3",
"through2": "2.0.1"
"through2": "^2.0.3"
},
"devDependencies": {
"ciao": "^0.6.0",

22
query/component_geocoding.js → query/structured_geocoding.js
Unescape View File

@ -6,21 +6,21 @@ var peliasQuery = require('pelias-query'),
//------------------------------
// general-purpose search query
//------------------------------
var componentQuery = new peliasQuery.layout.ComponentFallbackQuery();
var structuredQuery = new peliasQuery.layout.StructuredFallbackQuery();
// scoring boost
componentQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
componentQuery.score( peliasQuery.view.popularity_only_function );
componentQuery.score( peliasQuery.view.population_only_function );
structuredQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
structuredQuery.score( peliasQuery.view.popularity_only_function );
structuredQuery.score( peliasQuery.view.population_only_function );
// --------------------------------
// non-scoring hard filters
componentQuery.filter( peliasQuery.view.boundary_country );
componentQuery.filter( peliasQuery.view.boundary_circle );
componentQuery.filter( peliasQuery.view.boundary_rect );
componentQuery.filter( peliasQuery.view.sources );
componentQuery.filter( peliasQuery.view.layers );
componentQuery.filter( peliasQuery.view.categories );
structuredQuery.filter( peliasQuery.view.boundary_country );
structuredQuery.filter( peliasQuery.view.boundary_circle );
structuredQuery.filter( peliasQuery.view.boundary_rect );
structuredQuery.filter( peliasQuery.view.sources );
structuredQuery.filter( peliasQuery.view.layers );
structuredQuery.filter( peliasQuery.view.categories );
// --------------------------------
/**
@ -105,7 +105,7 @@ function generateQuery( clean ){
function getQuery(vs) {
return {
type: 'fallback',
body: componentQuery.render(vs)
body: structuredQuery.render(vs)
};
}

2
query/text_parser_addressit.js
Unescape View File

@ -32,8 +32,6 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
// ?
else {
logger.warn( 'chaos monkey asks: what happens now?' );
logger.warn( parsed_text );
try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
}
// ==== add parsed matches [address components] ====

16
routes/v1.js
Unescape View File

@ -8,7 +8,7 @@ var sanitizers = {
place: require('../sanitizer/place'),
search: require('../sanitizer/search'),
search_fallback: require('../sanitizer/search_fallback'),
component_geocoding: require('../sanitizer/component_geocoding'),
structured_geocoding: require('../sanitizer/structured_geocoding'),
reverse: require('../sanitizer/reverse'),
nearby: require('../sanitizer/nearby')
};
@ -30,14 +30,14 @@ var controllers = {
var queries = {
libpostal: require('../query/search'),
fallback_to_old_prod: require('../query/search_original'),
component_geocoding: require('../query/component_geocoding')
structured_geocoding: require('../query/structured_geocoding')
};
/** ----------------------- controllers ----------------------- **/
var postProc = {
trimByGranularity: require('../middleware/trimByGranularity'),
trimByGranularityComponent: require('../middleware/trimByGranularityComponent'),
trimByGranularityStructured: require('../middleware/trimByGranularityStructured'),
distances: require('../middleware/distance'),
confidenceScores: require('../middleware/confidenceScore'),
confidenceScoresFallback: require('../middleware/confidenceScoreFallback'),
@ -95,11 +95,11 @@ function addRoutes(app, peliasConfig) {
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
component: createRouter([
sanitizers.component_geocoding.middleware,
structured: createRouter([
sanitizers.structured_geocoding.middleware,
middleware.calcSize(),
controllers.search(peliasConfig, undefined, queries.component_geocoding),
postProc.trimByGranularityComponent(),
controllers.search(peliasConfig, undefined, queries.structured_geocoding),
postProc.trimByGranularityStructured(),
postProc.distances('focus.point.'),
postProc.confidenceScores(peliasConfig),
postProc.confidenceScoresFallback(),
@ -193,7 +193,7 @@ function addRoutes(app, peliasConfig) {
app.get ( base + 'autocomplete', routers.autocomplete );
app.get ( base + 'search', routers.search );
app.post( base + 'search', routers.search );
app.get ( base + 'beta/component', routers.component );
app.get ( base + 'search/structured', routers.structured );
app.get ( base + 'reverse', routers.reverse );
app.get ( base + 'nearby', routers.nearby );

2
sanitizer/_single_scalar_parameters.js
Unescape View File

@ -10,8 +10,10 @@ function sanitize( raw, clean ){
Object.keys(raw).forEach(function(key) {
if (_.isArray(raw[key])) {
messages.errors.push('\'' + key + '\' parameter can only have one value');
delete raw[key];
} else if (_.isObject(raw[key])) {
messages.errors.push('\'' + key + '\' parameter must be a scalar');
delete raw[key];
}
});

42
sanitizer/_synthesize_analysis.js
Unescape View File

@ -1,4 +1,5 @@
const _ = require('lodash');
const text_analyzer = require('pelias-text-analyzer');
const fields = {
'address': 'address',
@ -20,6 +21,19 @@ function isPostalCodeOnly(parsed_text) {
parsed_text.hasOwnProperty('postalcode');
}
// figure out which field contains the probable house number, prefer number
// libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street
// so because we're treating the entire field as a street address, it's safe
// to assume that an identified postcode is actually a house number.
function getHouseNumberField(analyzed_address) {
for (var field of ['number', 'postalcode']) {
if (analyzed_address.hasOwnProperty(field)) {
return field;
}
}
}
function sanitize( raw, clean ){
// error & warning messages
@ -43,6 +57,34 @@ function sanitize( raw, clean ){
`at least one of the following fields is required: ${Object.keys(fields).join(', ')}`);
}
if (clean.parsed_text.hasOwnProperty('address')) {
const analyzed_address = text_analyzer.parse(clean.parsed_text.address);
const house_number_field = getHouseNumberField(analyzed_address);
// if we're fairly certain that libpostal identified a house number
// (from either the house_number or postcode field), place it into the
// number field and remove the first instance of that value from address
// and assign to street
// eg - '1090 N Charlotte St' becomes number=1090 and street=N Charlotte St
if (house_number_field) {
clean.parsed_text.number = analyzed_address[house_number_field];
// remove the first instance of the number and trim whitespace
clean.parsed_text.street = _.trim(_.replace(clean.parsed_text.address, clean.parsed_text.number, ''));
} else {
// otherwise no house number was identifiable, so treat the entire input
// as a street
clean.parsed_text.street = clean.parsed_text.address;
}
// the address field no longer means anything since it's been parsed, so remove it
delete clean.parsed_text.address;
}
return messages;
}

2
sanitizer/reverse.js
Unescape View File

@ -2,8 +2,8 @@
var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
// depends on the layers and sources sanitizers, must be run after them

2
sanitizer/search.js
Unescape View File

@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
text: require('../sanitizer/_text'),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
size: require('../sanitizer/_size')(/* use defaults*/),

2
sanitizer/search_fallback.js
Unescape View File

@ -4,7 +4,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'),
};
var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
var logger = require('pelias-logger').get('api:controller:search_fallback');
var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' );
// middleware

2
sanitizer/component_geocoding.js → sanitizer/structured_geocoding.js
Unescape View File

@ -2,8 +2,8 @@ var type_mapping = require('../helper/type_mapping');
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
synthesize_analysis: require('../sanitizer/_synthesize_analysis'),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
size: require('../sanitizer/_size')(/* use defaults*/),

13
service/mget.js
Unescape View File

@ -11,7 +11,7 @@
**/
var peliasLogger = require( 'pelias-logger' ).get( 'service/mget' );
var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, query, cb ){
@ -24,8 +24,17 @@ function service( backend, query, cb ){
// query new backend
backend().client.mget( cmd, function( err, data ){
// log total ms elasticsearch reported the query took to execute
if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
}
// handle backend errors
if( err ){ return cb( err ); }
if( err ){
logger.error( 'backend error', err );
return cb( err );
}
// map returned documents
var docs = [];

15
service/search.js
Unescape View File

@ -5,18 +5,23 @@
**/
var peliasLogger = require( 'pelias-logger' ).get( 'service/search' );
var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, cmd, cb ){
// query new backend
backend().client.search( cmd, function( err, data ){
// handle backend errors
if( err ){ return cb( err ); }
// log total ms elasticsearch reported the query took to execute
peliasLogger.verbose( 'time elasticsearch reported:', data.took / 1000 );
if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
}
// handle backend errors
if( err ){
logger.error( 'backend error', err );
return cb( err );
}
// map returned documents
var docs = [];

30
test/ciao/autocomplete/non_scalar_parameter_array.coffee
Unescape View File

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/autocomplete?text=A&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

30
test/ciao/autocomplete/non_scalar_parameter_object.coffee
Unescape View File

@ -0,0 +1,30 @@
#> define parameter as object
path: '/v1/autocomplete?text=A&parameter[idx]=value'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

30
test/ciao/reverse/non_scalar_parameter_array.coffee
Unescape View File

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/reverse?point.lat=1&point.lon=1&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

2
test/ciao/reverse/non_scalar_parameter.coffee → test/ciao/reverse/non_scalar_parameter_object.coffee
Unescape View File

@ -1,5 +1,5 @@
#> set size
#> define parameter as object
path: '/v1/reverse?point.lat=1&point.lon=1&parameter[idx]=value'
#? 200 ok

2
test/ciao/search/address_parsing.coffee
Unescape View File

@ -35,7 +35,7 @@ json.geocoding.query['size'].should.eql 10
#? address parsing
json.geocoding.query.parsed_text['number'].should.eql '30'
json.geocoding.query.parsed_text['street'].should.eql 'w 26th st'
json.geocoding.query.parsed_text['state'].should.eql 'ny'
json.geocoding.query.parsed_text['state'].should.eql 'NY'
json.features[0].properties.confidence.should.eql 1
json.features[0].properties.match_type.should.eql "exact"

30
test/ciao/search/non_scalar_parameter_array.coffee
Unescape View File

@ -0,0 +1,30 @@
#> define two sources
path: '/v1/search?text=A&sources=A&sources=B'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'sources\' parameter can only have one value' ]

30
test/ciao/search/non_scalar_parameter_object.coffee
Unescape View File

@ -0,0 +1,30 @@
#> define parameter as object
path: '/v1/search?text=A&parameter[idx]=value'
#? 200 ok
response.statusCode.should.be.equal 400
response.should.have.header 'charset', 'utf8'
response.should.have.header 'content-type', 'application/json; charset=utf-8'
#? valid geocoding block
should.exist json.geocoding
should.exist json.geocoding.version
should.exist json.geocoding.attribution
should.exist json.geocoding.query
should.exist json.geocoding.engine
should.exist json.geocoding.engine.name
should.exist json.geocoding.engine.author
should.exist json.geocoding.engine.version
should.exist json.geocoding.timestamp
#? valid geojson
json.type.should.be.equal 'FeatureCollection'
json.features.should.be.instanceof Array
#? expected warnings
should.not.exist json.geocoding.warnings
#? expected errors
should.exist json.geocoding.errors
json.geocoding.errors.should.eql [ '\'parameter\' parameter must be a scalar' ]

0
test/unit/fixture/component_geocoding/boundary_country.json → test/unit/fixture/structured_geocoding/boundary_country.json
Unescape View File

191
test/unit/fixture/component_geocoding/fallback.json → test/unit/fixture/structured_geocoding/fallback.json
Unescape View File

@ -6,6 +6,197 @@
"query": {
"bool": {
"should": [
{
"bool": {
"_name": "fallback.address",
"must": [
{
"match_phrase": {
"address_parts.number": "number value"
}
},
{
"match_phrase": {
"address_parts.street": "street value"
}
},
{
"multi_match": {
"query": "neighbourhood value",
"type": "phrase",
"fields": [
"parent.neighbourhood",
"parent.neighbourhood_a"
]
}
},
{
"multi_match": {
"query": "borough value",
"type": "phrase",
"fields": [
"parent.borough",
"parent.borough_a"
]
}
},
{
"multi_match": {
"query": "city value",
"type": "phrase",
"fields": [
"parent.locality",
"parent.locality_a",
"parent.localadmin",
"parent.localadmin_a"
]
}
},
{
"multi_match": {
"query": "county value",
"type": "phrase",
"fields": [
"parent.county",
"parent.county_a",
"parent.macrocounty",
"parent.macrocounty_a"
]
}
},
{
"multi_match": {
"query": "state value",
"type": "phrase",
"fields": [
"parent.region",
"parent.region_a",
"parent.macroregion",
"parent.macroregion_a"
]
}
},
{
"multi_match": {
"query": "country value",
"type": "phrase",
"fields": [
"parent.country",
"parent.country_a",
"parent.dependency",
"parent.dependency_a"
]
}
}
],
"should": [
{
"match_phrase": {
"address_parts.zip": "postalcode value"
}
}
],
"filter": {
"term": {
"layer": "address"
}
},
"boost": 10
}
},
{
"bool": {
"_name": "fallback.street",
"must": [
{
"match_phrase": {
"address_parts.street": "street value"
}
},
{
"multi_match": {
"query": "neighbourhood value",
"type": "phrase",
"fields": [
"parent.neighbourhood",
"parent.neighbourhood_a"
]
}
},
{
"multi_match": {
"query": "borough value",
"type": "phrase",
"fields": [
"parent.borough",
"parent.borough_a"
]
}
},
{
"multi_match": {
"query": "city value",
"type": "phrase",
"fields": [
"parent.locality",
"parent.locality_a",
"parent.localadmin",
"parent.localadmin_a"
]
}
},
{
"multi_match": {
"query": "county value",
"type": "phrase",
"fields": [
"parent.county",
"parent.county_a",
"parent.macrocounty",
"parent.macrocounty_a"
]
}
},
{
"multi_match": {
"query": "state value",
"type": "phrase",
"fields": [
"parent.region",
"parent.region_a",
"parent.macroregion",
"parent.macroregion_a"
]
}
},
{
"multi_match": {
"query": "country value",
"type": "phrase",
"fields": [
"parent.country",
"parent.country_a",
"parent.dependency",
"parent.dependency_a"
]
}
}
],
"should": [
{
"match_phrase": {
"address_parts.zip": "postalcode value"
}
}
],
"filter": {
"term": {
"layer": "street"
}
},
"boost": 5
}
},
{
"bool": {
"_name": "fallback.neighbourhood",

0
test/unit/fixture/component_geocoding/linguistic_bbox.json → test/unit/fixture/structured_geocoding/linguistic_bbox.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_focus.json → test/unit/fixture/structured_geocoding/linguistic_focus.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_focus_bbox.json → test/unit/fixture/structured_geocoding/linguistic_focus_bbox.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_focus_null_island.json → test/unit/fixture/structured_geocoding/linguistic_focus_null_island.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_only.json → test/unit/fixture/structured_geocoding/linguistic_only.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_viewport.json → test/unit/fixture/structured_geocoding/linguistic_viewport.json
Unescape View File

0
test/unit/fixture/component_geocoding/linguistic_viewport_min_diagonal.json → test/unit/fixture/structured_geocoding/linguistic_viewport_min_diagonal.json
Unescape View File

0
test/unit/fixture/component_geocoding/with_source_filtering.json → test/unit/fixture/structured_geocoding/with_source_filtering.json
Unescape View File

26
test/unit/middleware/trimByGranularityComponent.js → test/unit/middleware/trimByGranularityStructured.js
Unescape View File

@ -1,4 +1,4 @@
var trimByGranularity = require('../../../middleware/trimByGranularityComponent')();
var trimByGranularity = require('../../../middleware/trimByGranularityStructured')();
module.exports.tests = {};
@ -19,8 +19,8 @@ module.exports.tests.trimByGranularity = function(test, common) {
data: [
{ name: 'venue 1', _matched_queries: ['fallback.venue'] },
{ name: 'venue 2', _matched_queries: ['fallback.venue'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -55,9 +55,9 @@ module.exports.tests.trimByGranularity = function(test, common) {
var res = {
data: [
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -73,13 +73,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
};
var expected_data = [
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] }
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] }
];
function testIt() {
trimByGranularity(req, res, function() {
t.deepEquals(res.data, expected_data, 'only street records should be here');
t.deepEquals(res.data, expected_data, 'only address records should be here');
t.end();
});
}
@ -87,13 +87,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
testIt();
});
test('all records with fallback.* matched_queries name should retain only address when they are most granular', function(t) {
test('all records with fallback.* matched_queries name should retain only street when they are most granular', function(t) {
var req = { clean: {} };
var res = {
data: [
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
{ name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
{ name: 'borough 1', _matched_queries: ['fallback.borough'] },
{ name: 'locality 1', _matched_queries: ['fallback.locality'] },
@ -109,13 +109,13 @@ module.exports.tests.trimByGranularity = function(test, common) {
};
var expected_data = [
{ name: 'address 1', _matched_queries: ['fallback.address'] },
{ name: 'address 2', _matched_queries: ['fallback.address'] },
{ name: 'street 1', _matched_queries: ['fallback.street'] },
{ name: 'street 2', _matched_queries: ['fallback.street'] },
];
function testIt() {
trimByGranularity(req, res, function() {
t.deepEquals(res.data, expected_data, 'only address records should be here');
t.deepEquals(res.data, expected_data, 'only street records should be here');
t.end();
});
}

33
test/unit/query/component_geocoding.js → test/unit/query/structured_geocoding.js
Unescape View File

@ -1,4 +1,4 @@
var generate = require('../../../query/component_geocoding');
var generate = require('../../../query/structured_geocoding');
var fs = require('fs');
module.exports.tests = {};
@ -14,7 +14,6 @@ module.exports.tests.query = function(test, common) {
test('valid search + focus + bbox', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test',
querySize: 10,
@ -29,7 +28,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus_bbox');
var expected = require('../fixture/structured_geocoding/linguistic_focus_bbox');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox');
@ -39,7 +38,6 @@ module.exports.tests.query = function(test, common) {
test('valid search + bbox', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test',
querySize: 10,
@ -53,7 +51,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_bbox');
var expected = require('../fixture/structured_geocoding/linguistic_bbox');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_bbox');
@ -63,7 +61,6 @@ module.exports.tests.query = function(test, common) {
test('valid lingustic-only search', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
layers: ['test']
@ -72,7 +69,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_only');
var expected = require('../fixture/structured_geocoding/linguistic_only');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_only');
@ -82,7 +79,6 @@ module.exports.tests.query = function(test, common) {
test('search search + focus', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
@ -92,7 +88,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus');
var expected = require('../fixture/structured_geocoding/linguistic_focus');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus');
@ -102,7 +98,6 @@ module.exports.tests.query = function(test, common) {
test('search search + viewport', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.viewport.min_lat': 28.49136,
@ -115,7 +110,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_viewport');
var expected = require('../fixture/structured_geocoding/linguistic_viewport');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_viewport');
@ -127,7 +122,6 @@ module.exports.tests.query = function(test, common) {
test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.viewport.min_lat': 28.49135,
@ -140,7 +134,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_viewport_min_diagonal');
var expected = require('../fixture/structured_geocoding/linguistic_viewport_min_diagonal');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'valid search query');
@ -150,7 +144,6 @@ module.exports.tests.query = function(test, common) {
test('search search + focus on null island', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
'focus.point.lat': 0, 'focus.point.lon': 0,
@ -160,7 +153,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/linguistic_focus_null_island');
var expected = require('../fixture/structured_geocoding/linguistic_focus_null_island');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island');
@ -187,7 +180,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse(JSON.stringify(query));
var expected = require('../fixture/component_geocoding/fallback');
var expected = require('../fixture/structured_geocoding/fallback');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'fallbackQuery');
@ -198,7 +191,6 @@ module.exports.tests.query = function(test, common) {
test('valid boundary.country search', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
text: 'test', querySize: 10,
layers: ['test'],
@ -208,7 +200,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/boundary_country');
var expected = require('../fixture/structured_geocoding/boundary_country');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search: valid boundary.country query');
@ -218,7 +210,6 @@ module.exports.tests.query = function(test, common) {
test('valid sources filter', function(t) {
var clean = {
parsed_text: {
street: 'street value'
},
'text': 'test',
'sources': ['test_source']
@ -227,7 +218,7 @@ module.exports.tests.query = function(test, common) {
var query = generate(clean);
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/component_geocoding/with_source_filtering');
var expected = require('../fixture/structured_geocoding/with_source_filtering');
t.deepEqual(compiled.type, 'fallback', 'query type set');
t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering');
@ -238,7 +229,7 @@ module.exports.tests.query = function(test, common) {
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('component_geocoding query ' + name, testFunction);
return tape('structured_geocoding query ' + name, testFunction);
}
for( var testCase in module.exports.tests ){

6
test/unit/run.js
Unescape View File

@ -30,15 +30,15 @@ var tests = [
require('./middleware/sendJSON'),
require('./middleware/normalizeParentIds'),
require('./middleware/trimByGranularity'),
require('./middleware/trimByGranularityComponent'),
require('./middleware/trimByGranularityStructured'),
require('./query/autocomplete'),
require('./query/autocomplete_defaults'),
require('./query/component_geocoding'),
require('./query/search_defaults'),
require('./query/reverse_defaults'),
require('./query/reverse'),
require('./query/search'),
require('./query/search_original'),
require('./query/structured_geocoding'),
require('./query/text_parser'),
require('./sanitizer/_boundary_country'),
require('./sanitizer/_flag_bool'),
@ -61,7 +61,7 @@ var tests = [
require('./sanitizer/nearby'),
require('./src/backend'),
require('./sanitizer/autocomplete'),
require('./sanitizer/component_geocoding'),
require('./sanitizer/structured_geocoding'),
require('./sanitizer/place'),
require('./sanitizer/reverse'),
require('./sanitizer/sanitizeAll'),

9
test/unit/sanitizer/_single_scalar_parameters.js
Unescape View File

@ -18,6 +18,11 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
],
warnings: []
});
// erroneous params should be deleted to avoid middleware errors
t.false(raw.arrayParameter1);
t.false(raw.arrayParameter2);
t.end();
});
@ -36,6 +41,10 @@ module.exports.tests.single_scalar_parameters = function(test, common) {
],
warnings: []
});
// erroneous params should be deleted to avoid middleware errors
t.false(raw.objectParameter1);
t.false(raw.objectParameter2);
t.end();
});

124
test/unit/sanitizer/_synthesize_analysis.js
Unescape View File

@ -1,13 +1,18 @@
const sanitizer = require('../../../sanitizer/_synthesize_analysis');
const _ = require('lodash');
const proxyquire = require('proxyquire').noCallThru();
module.exports.tests = {};
module.exports.tests.text_parser = function(test, common) {
test('all variables should be parsed', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {
query: ' \t query \t value \t ',
address: ' \t address \t value \t ',
neighbourhood: ' \t neighbourhood \t value \t ',
borough: ' \t borough \t value \t ',
locality: ' \t locality \t value \t ',
@ -21,7 +26,6 @@ module.exports.tests.text_parser = function(test, common) {
const expected_clean = {
parsed_text: {
address: 'address value',
neighbourhood: 'neighbourhood value',
borough: 'borough value',
city: 'locality value',
@ -42,6 +46,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('non-string and blank string values should be treated as not supplied', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
// helper to return a random value that's considered invalid
function getInvalidValue() {
return _.sample([{}, [], false, '', ' \t ', 17, undefined]);
@ -75,6 +85,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('no supplied fields should return error', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {};
const clean = {};
@ -92,6 +108,12 @@ module.exports.tests.text_parser = function(test, common) {
});
test('postalcode-only parsed_text should return error', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.fail('parse should not have been called');
}
}});
const raw = {
postalcode: 'postalcode value'
};
@ -113,6 +135,102 @@ module.exports.tests.text_parser = function(test, common) {
});
test('text_analyzer identifying house number should extract it and street', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Number Value Street Value Number Value');
return {
number: 'Number Value'
};
}
}});
const raw = {
address: 'Number Value Street Value Number Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
number: 'Number Value',
street: 'Street Value Number Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
test('text_analyzer identifying postalcode but not house number should assign to number and remove from address', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Number Value Street Value Number Value');
return {
postalcode: 'Number Value'
};
}
}});
const raw = {
address: 'Number Value Street Value Number Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
number: 'Number Value',
street: 'Street Value Number Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
test('text_analyzer not revealing possible number should move address to street', function(t) {
var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', {
'pelias-text-analyzer': { parse: function(query) {
t.equals(query, 'Street Value');
return {};
}
}});
const raw = {
address: 'Street Value'
};
const clean = {};
const expected_clean = {
parsed_text: {
street: 'Street Value'
}
};
const messages = sanitizer(raw, clean);
t.deepEquals(clean, expected_clean);
t.deepEquals(messages.errors, [], 'no errors');
t.deepEquals(messages.warnings, [], 'no warnings');
t.end();
});
};
module.exports.all = function (tape, common) {

2
test/unit/sanitizer/nearby.js
Unescape View File

@ -30,7 +30,7 @@ module.exports.tests.interface = function(test, common) {
module.exports.tests.sanitizers = function(test, common) {
test('check sanitizer list', function (t) {
var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country', 'categories'];
t.deepEqual(Object.keys(nearby.sanitizer_list), expected);
t.end();

2
test/unit/sanitizer/reverse.js
Unescape View File

@ -36,7 +36,7 @@ module.exports.tests.interface = function(test, common) {
module.exports.tests.sanitizers = function(test, common) {
test('check sanitizer list', function (t) {
var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
var expected = ['singleScalarParameters', 'quattroshapes_deprecation', 'layers',
'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country'];
t.deepEqual(Object.keys(reverse.sanitizer_list), expected);
t.end();

2
test/unit/sanitizer/search.js
Unescape View File

@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
});
var expected_sanitizers = [
'_deprecate_quattroshapes',
'_single_scalar_parameters',
'_deprecate_quattroshapes',
'_text',
'_iso2_to_iso3',
'_size',

6
test/unit/sanitizer/component_geocoding.js → test/unit/sanitizer/structured_geocoding.js
Unescape View File

@ -8,7 +8,7 @@ module.exports.tests.sanitize = function(test, common) {
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/component_geocoding', {
var search = proxyquire('../../../sanitizer/structured_geocoding', {
'../sanitizer/_deprecate_quattroshapes': function() {
called_sanitizers.push('_deprecate_quattroshapes');
return { errors: [], warnings: [] };
@ -82,8 +82,8 @@ module.exports.tests.sanitize = function(test, common) {
});
var expected_sanitizers = [
'_deprecate_quattroshapes',
'_single_scalar_parameters',
'_deprecate_quattroshapes',
'_synthesize_analysis',
'_iso2_to_iso3',
'_size',
@ -109,7 +109,7 @@ module.exports.tests.sanitize = function(test, common) {
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANTIZE /component ' + name, testFunction);
return tape('SANTIZE /structured ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
Write Preview
Loading…
Cancel
Save