pelias
/
api
mirror of https://github.com/pelias/api.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

Merge pull request #537 from pelias/staging 

Merge staging into production
pull/745/head
Diana Shkolnikov 9 years ago
parent
dfe924c4ca 181a3f2399
commit
d2e714733b
12 changed files with 334 additions and 191 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      controller/place.js
  2. 4
      controller/search.js
  3. 48
      middleware/sendJSON.js
  4. 1
      package.json
  5. 18
      query/search.js
  6. 10
      query/search_defaults.js
  7. 2
      test/ciao_test_data.js
  8. 65
      test/unit/fixture/search_full_address.js
  9. 71
      test/unit/fixture/search_partial_address.js
  10. 65
      test/unit/fixture/search_regions_address.js
  11. 236
      test/unit/middleware/sendJSON.js
  12. 1
      test/unit/run.js

4
controller/place.js
Unescape View File

@ -22,7 +22,7 @@ function setup( backend ){
}; };
}); });
logger.debug( '[ES req]', JSON.stringify(query) ); logger.debug( '[ES req]', query );
service.mget( backend, query, function( err, docs ) { service.mget( backend, query, function( err, docs ) {
console.log('err:' + err); console.log('err:' + err);
@ -35,7 +35,7 @@ function setup( backend ){
else { else {
res.data = docs; res.data = docs;
} }
logger.debug('[ES response]', JSON.stringify(docs)); logger.debug('[ES response]', docs);
next(); next();
}); });

4
controller/search.js
Unescape View File

@ -31,7 +31,7 @@ function setup( backend, query ){
cmd.type = req.clean.layers; cmd.type = req.clean.layers;
} }
logger.debug( '[ES req]', JSON.stringify(cmd) ); logger.debug( '[ES req]', cmd );
// query backend // query backend
service.search( backend, cmd, function( err, docs, meta ){ service.search( backend, cmd, function( err, docs, meta ){
@ -49,7 +49,7 @@ function setup( backend, query ){
res.data = docs; res.data = docs;
res.meta = meta; res.meta = meta;
} }
logger.debug('[ES response]', JSON.stringify(docs)); logger.debug('[ES response]', docs);
next(); next();
}); });

48
middleware/sendJSON.js
Unescape View File

@ -1,4 +1,12 @@
var check = require('check-types'); var check = require('check-types'),
es = require('elasticsearch'),
exceptions = require('elasticsearch-exceptions/lib/exceptions/SupportedExceptions');
// create a list of regular expressions to match against.
// note: list created when the server starts up; for performance reasons.
var exceptionRegexList = exceptions.map( function( exceptionName ){
return new RegExp( '^' + exceptionName );
});
function sendJSONResponse(req, res, next) { function sendJSONResponse(req, res, next) {
@ -8,10 +16,11 @@ function sendJSONResponse(req, res, next) {
} }
// default status // default status
var statusCode = 200; var statusCode = 200; // 200 OK
// vary status code whenever an error was reported // vary status code whenever an error was reported
var geocoding = res.body.geocoding; var geocoding = res.body.geocoding;
if( check.array( geocoding.errors ) && geocoding.errors.length ){ if( check.array( geocoding.errors ) && geocoding.errors.length ){
// default status for errors is 400 Bad Request // default status for errors is 400 Bad Request
@ -19,18 +28,33 @@ function sendJSONResponse(req, res, next) {
// iterate over all reported errors // iterate over all reported errors
geocoding.errors.forEach( function( err ){ geocoding.errors.forEach( function( err ){
// custom status codes for instances of the Error() object. // custom status codes for instances of the Error() object.
if( err instanceof Error ){ if( err instanceof Error ){
// we can extract the error type from the constructor name /*
switch( err.constructor.name ){ elasticsearch errors
// elasticsearch errors see: https://github.com/elastic/elasticsearch-js/blob/master/src/lib/errors.js
// see: https://github.com/elastic/elasticsearch-js/blob/master/src/lib/errors.js
case 'RequestTimeout': statusCode = 408; break; // 408 Request Timeout 408 Request Timeout
case 'NoConnections': statusCode = 502; break; // 502 Bad Gateway 500 Internal Server Error
case 'ConnectionFault': statusCode = 502; break; // 502 Bad Gateway 502 Bad Gateway
case 'Serialization': statusCode = 500; break; // 500 Internal Server Error */
case 'Generic': statusCode = 500; break; // 500 Internal Server Error if( err instanceof es.errors.RequestTimeout ){ statusCode = Math.max( statusCode, 408 ); }
default: statusCode = 500; // 500 Internal Server Error else if( err instanceof es.errors.NoConnections ){ statusCode = Math.max( statusCode, 502 ); }
else if( err instanceof es.errors.ConnectionFault ){ statusCode = Math.max( statusCode, 502 ); }
else { statusCode = Math.max( statusCode, 500 ); }
/*
some elasticsearch errors are only returned as strings (not instances of Error).
in this case we (unfortunately) need to match the exception at position 0 inside the string.
*/
} else if( check.string( err ) ){
for( var i=0; i<exceptionRegexList.length; i++ ){
// check error string against a list of known elasticsearch exceptions
if( err.match( exceptionRegexList[i] ) ){
statusCode = Math.max( statusCode, 500 );
break; // break on first match
}
} }
} }
}); });

1
package.json
Unescape View File

@ -39,6 +39,7 @@
"check-types": "^6.0.0", "check-types": "^6.0.0",
"cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",
"elasticsearch": "^11.0.0", "elasticsearch": "^11.0.0",
"elasticsearch-exceptions": "0.0.4",
"express": "^4.8.8", "express": "^4.8.8",
"express-http-proxy": "^0.6.0", "express-http-proxy": "^0.6.0",
"extend": "3.0.0", "extend": "3.0.0",

18
query/search.js
Unescape View File

@ -3,6 +3,13 @@ var peliasQuery = require('pelias-query'),
textParser = require('./text_parser'), textParser = require('./text_parser'),
check = require('check-types'), check = require('check-types'),
geolib = require('geolib'); geolib = require('geolib');
var placeTypes = require('../helper/placeTypes');
// region_a is also an admin field. addressit tries to detect
// region_a, in which case we use a match query specifically for it.
// but address it doesn't know about all of them so it helps to search
// against this with the other admin parts as a fallback
var adminFields = placeTypes.concat(['region_a']);
//------------------------------ //------------------------------
// general-purpose search query // general-purpose search query
@ -25,15 +32,12 @@ query.score( peliasQuery.view.address('street') );
query.score( peliasQuery.view.address('postcode') ); query.score( peliasQuery.view.address('postcode') );
// admin components // admin components
query.score( peliasQuery.view.admin('country') ); // country_a and region_a are left as matches here because the text-analyzer
// can sometimes detect them, in which case a query more specific than a
// multi_match is appropriate.
query.score( peliasQuery.view.admin('country_a') ); query.score( peliasQuery.view.admin('country_a') );
query.score( peliasQuery.view.admin('region') );
query.score( peliasQuery.view.admin('region_a') ); query.score( peliasQuery.view.admin('region_a') );
query.score( peliasQuery.view.admin('county') ); query.score( peliasQuery.view.admin_multi_match(adminFields), 'peliasAdmin' );
query.score( peliasQuery.view.admin('borough') );
query.score( peliasQuery.view.admin('localadmin') );
query.score( peliasQuery.view.admin('locality') );
query.score( peliasQuery.view.admin('neighbourhood') );
// non-scoring hard filters // non-scoring hard filters
query.filter( peliasQuery.view.boundary_circle ); query.filter( peliasQuery.view.boundary_circle );

10
query/search_defaults.js
Unescape View File

@ -52,23 +52,23 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'admin:country_a:analyzer': 'standard', 'admin:country_a:analyzer': 'standard',
'admin:country_a:field': 'parent.country_a', 'admin:country_a:field': 'parent.country_a',
'admin:country_a:boost': 5, 'admin:country_a:boost': 1,
'admin:country:analyzer': 'peliasAdmin', 'admin:country:analyzer': 'peliasAdmin',
'admin:country:field': 'parent.country', 'admin:country:field': 'parent.country',
'admin:country:boost': 4, 'admin:country:boost': 1,
'admin:region:analyzer': 'peliasAdmin', 'admin:region:analyzer': 'peliasAdmin',
'admin:region:field': 'parent.region', 'admin:region:field': 'parent.region',
'admin:region:boost': 3, 'admin:region:boost': 1,
'admin:region_a:analyzer': 'peliasAdmin', 'admin:region_a:analyzer': 'peliasAdmin',
'admin:region_a:field': 'parent.region_a', 'admin:region_a:field': 'parent.region_a',
'admin:region_a:boost': 3, 'admin:region_a:boost': 1,
'admin:county:analyzer': 'peliasAdmin', 'admin:county:analyzer': 'peliasAdmin',
'admin:county:field': 'parent.county', 'admin:county:field': 'parent.county',
'admin:county:boost': 2, 'admin:county:boost': 1,
'admin:localadmin:analyzer': 'peliasAdmin', 'admin:localadmin:analyzer': 'peliasAdmin',
'admin:localadmin:field': 'parent.localadmin', 'admin:localadmin:field': 'parent.localadmin',

2
test/ciao_test_data.js
Unescape View File

@ -54,5 +54,5 @@ actions.push( function( done ){
// perform all actions in series // perform all actions in series
async.series( actions, function( err, resp ){ async.series( actions, function( err, resp ){
console.log('test data inported'); console.log('test data imported');
}); });

65
test/unit/fixture/search_full_address.js
Unescape View File

@ -99,14 +99,6 @@ module.exports = {
'analyzer': vs['address:postcode:analyzer'] 'analyzer': vs['address:postcode:analyzer']
} }
} }
}, {
'match': {
'parent.country': {
'query': 'new york',
'boost': vs['admin:country:boost'],
'analyzer': vs['admin:country:analyzer']
}
}
}, { }, {
'match': { 'match': {
'parent.country_a': { 'parent.country_a': {
@ -115,14 +107,6 @@ module.exports = {
'analyzer': vs['admin:country_a:analyzer'] 'analyzer': vs['admin:country_a:analyzer']
} }
} }
}, {
'match': {
'parent.region': {
'query': 'new york',
'boost': vs['admin:region:boost'],
'analyzer': vs['admin:region:analyzer']
}
}
}, { }, {
'match': { 'match': {
'parent.region_a': { 'parent.region_a': {
@ -132,44 +116,19 @@ module.exports = {
} }
} }
}, { }, {
'match': { 'multi_match': {
'parent.county': { 'fields': [
'query': 'new york', 'parent.country^1',
'boost': vs['admin:county:boost'], 'parent.region^1',
'analyzer': vs['admin:county:analyzer'] 'parent.county^1',
} 'parent.localadmin^1',
} 'parent.locality^1',
}, { 'parent.borough^1',
'match': { 'parent.neighbourhood^1',
'parent.borough': { 'parent.region_a^1'
'query': 'new york', ],
'boost': vs['admin:borough:boost'],
'analyzer': vs['admin:borough:analyzer']
}
}
}, {
'match': {
'parent.localadmin': {
'query': 'new york',
'boost': vs['admin:localadmin:boost'],
'analyzer': vs['admin:localadmin:analyzer']
}
}
}, {
'match': {
'parent.locality': {
'query': 'new york',
'boost': vs['admin:locality:boost'],
'analyzer': vs['admin:locality:analyzer']
}
}
}, {
'match': {
'parent.neighbourhood': {
'query': 'new york', 'query': 'new york',
'boost': vs['admin:neighbourhood:boost'], 'analyzer': 'peliasAdmin'
'analyzer': vs['admin:neighbourhood:analyzer']
}
} }
}] }]
} }

71
test/unit/fixture/search_partial_address.js
Unescape View File

@ -75,69 +75,28 @@ module.exports = {
'weight': 2 'weight': 2
}] }]
} }
},{
'match': {
'parent.country': {
'query': 'new york',
'boost': vs['admin:country:boost'],
'analyzer': vs['admin:country:analyzer']
}
}
}, {
'match': {
'parent.region': {
'query': 'new york',
'boost': vs['admin:region:boost'],
'analyzer': vs['admin:region:analyzer']
}
}
}, { }, {
'match': { 'match': {
'parent.region_a': { 'parent.region_a': {
'query': 'new york', 'analyzer': 'peliasAdmin',
'boost': vs['admin:region_a:boost'], 'boost': 1,
'analyzer': vs['admin:region_a:analyzer'] 'query': 'new york'
}
}
}, {
'match': {
'parent.county': {
'query': 'new york',
'boost': vs['admin:county:boost'],
'analyzer': vs['admin:county:analyzer']
} }
} }
}, { }, {
'match': { 'multi_match': {
'parent.borough': { 'fields': [
'parent.country^1',
'parent.region^1',
'parent.county^1',
'parent.localadmin^1',
'parent.locality^1',
'parent.borough^1',
'parent.neighbourhood^1',
'parent.region_a^1'
],
'query': 'new york', 'query': 'new york',
'boost': vs['admin:borough:boost'], 'analyzer': 'peliasAdmin'
'analyzer': vs['admin:borough:analyzer']
}
}
}, {
'match': {
'parent.localadmin': {
'query': 'new york',
'boost': vs['admin:localadmin:boost'],
'analyzer': vs['admin:localadmin:analyzer']
}
}
}, {
'match': {
'parent.locality': {
'query': 'new york',
'boost': vs['admin:locality:boost'],
'analyzer': vs['admin:locality:analyzer']
}
}
}, {
'match': {
'parent.neighbourhood': {
'query': 'new york',
'boost': vs['admin:neighbourhood:boost'],
'analyzer': vs['admin:neighbourhood:analyzer']
}
} }
}] }]
} }

65
test/unit/fixture/search_regions_address.js
Unescape View File

@ -91,22 +91,6 @@ module.exports = {
'analyzer': vs['address:street:analyzer'] 'analyzer': vs['address:street:analyzer']
} }
} }
}, {
'match': {
'parent.country': {
'query': 'manhattan',
'boost': vs['admin:country:boost'],
'analyzer': vs['admin:country:analyzer']
}
}
}, {
'match': {
'parent.region': {
'query': 'manhattan',
'boost': vs['admin:region:boost'],
'analyzer': vs['admin:region:analyzer']
}
}
}, { }, {
'match': { 'match': {
'parent.region_a': { 'parent.region_a': {
@ -116,44 +100,19 @@ module.exports = {
} }
} }
}, { }, {
'match': { 'multi_match': {
'parent.county': { 'fields': [
'query': 'manhattan', 'parent.country^1',
'boost': vs['admin:county:boost'], 'parent.region^1',
'analyzer': vs['admin:county:analyzer'] 'parent.county^1',
} 'parent.localadmin^1',
} 'parent.locality^1',
}, { 'parent.borough^1',
'match': { 'parent.neighbourhood^1',
'parent.borough': { 'parent.region_a^1'
'query': 'manhattan', ],
'boost': vs['admin:borough:boost'],
'analyzer': vs['admin:borough:analyzer']
}
}
}, {
'match': {
'parent.localadmin': {
'query': 'manhattan',
'boost': vs['admin:localadmin:boost'],
'analyzer': vs['admin:localadmin:analyzer']
}
}
}, {
'match': {
'parent.locality': {
'query': 'manhattan',
'boost': vs['admin:locality:boost'],
'analyzer': vs['admin:locality:analyzer']
}
}
}, {
'match': {
'parent.neighbourhood': {
'query': 'manhattan', 'query': 'manhattan',
'boost': vs['admin:neighbourhood:boost'], 'analyzer': 'peliasAdmin'
'analyzer': vs['admin:neighbourhood:analyzer']
}
} }
}] }]
} }

236
test/unit/middleware/sendJSON.js
Unescape View File

@ -0,0 +1,236 @@
var es = require('elasticsearch'),
middleware = require('../../../middleware/sendJSON');
module.exports.tests = {};
module.exports.tests.invalid = function(test, common) {
test('invalid $res', function(t) {
var res;
middleware(null, res, function () {
t.pass('next() called.');
t.end();
});
});
test('invalid $res.body', function(t) {
var res = { body: 1 };
middleware(null, res, function () {
t.pass('next() called.');
t.end();
});
});
test('invalid $res.body.geocoding', function(t) {
var res = { body: { geocoding: 1 } };
middleware(null, res, function () {
t.pass('next() called.');
t.end();
});
});
};
module.exports.tests.default_status = function(test, common) {
test('no errors', function(t) {
var res = { body: { geocoding: {} } };
res.status = function( code ){
return { json: function( body ){
t.equal( code, 200, '200 OK' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
test('empty errors array', function(t) {
var res = { body: { geocoding: {}, errors: [] } };
res.status = function( code ){
return { json: function( body ){
t.equal( code, 200, '200 OK' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.default_error_status = function(test, common) {
test('default error code', function(t) {
var res = { body: { geocoding: {
errors: [ 'an error' ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 400, '400 Bad Request' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.generic_server_error = function(test, common) {
test('generic server error', function(t) {
var res = { body: { geocoding: {
errors: [ new Error('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 500, 'Internal Server Error' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.generic_elasticsearch_error = function(test, common) {
test('generic elasticsearch error', function(t) {
var res = { body: { geocoding: {
errors: [ new es.errors.Generic('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 500, 'Internal Server Error' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.request_timeout = function(test, common) {
test('request timeout', function(t) {
var res = { body: { geocoding: {
errors: [ new es.errors.RequestTimeout('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 408, 'Request Timeout' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.no_connections = function(test, common) {
test('no connections', function(t) {
var res = { body: { geocoding: {
errors: [ new es.errors.NoConnections('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 502, 'Bad Gateway' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.connection_fault = function(test, common) {
test('connection fault', function(t) {
var res = { body: { geocoding: {
errors: [ new es.errors.ConnectionFault('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 502, 'Bad Gateway' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.serialization = function(test, common) {
test('serialization', function(t) {
var res = { body: { geocoding: {
errors: [ new es.errors.Serialization('an error') ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 500, 'Internal Server Error' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.search_phase_execution_exception = function(test, common) {
test('search phase execution exception', function(t) {
var res = { body: { geocoding: {
errors: [ 'SearchPhaseExecutionException[ foo ]' ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 500, 'Internal Server Error' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.tests.unknown_exception = function(test, common) {
test('unknown exception', function(t) {
var res = { body: { geocoding: {
errors: [ 'MadeUpExceptionName[ foo ]' ]
}}};
res.status = function( code ){
return { json: function( body ){
t.equal( code, 400, '400 Bad Request' );
t.deepEqual( body, res.body, 'body set' );
t.end();
}};
};
middleware(null, res);
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('[middleware] sendJSON: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

1
test/unit/run.js
Unescape View File

@ -27,6 +27,7 @@ var tests = [
require('./middleware/localNamingConventions'), require('./middleware/localNamingConventions'),
require('./middleware/dedupe'), require('./middleware/dedupe'),
require('./middleware/parseBBox'), require('./middleware/parseBBox'),
require('./middleware/sendJSON'),
require('./middleware/normalizeParentIds'), require('./middleware/normalizeParentIds'),
require('./query/autocomplete'), require('./query/autocomplete'),
require('./query/autocomplete_defaults'), require('./query/autocomplete_defaults'),

Write Preview
Loading…
Cancel
Save