pelias
/
api
mirror of https://github.com/pelias/api.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

Merge pull request #348 from pelias/master 

Production merge 10/14
pull/364/merge
Julian Simioni 9 years ago
parent
fe171d2211 d732021d5e
commit
fc9f756b17
11 changed files with 481 additions and 99 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      README.md
  2. 33
      controller/place.js
  3. 29
      query/search.js
  4. 16
      sanitiser/_geo_common.js
  5. 14
      sanitiser/_geo_search.js
  6. 4
      test/unit/controller/place.js
  7. 136
      test/unit/fixture/search_linguistic_viewport.js
  8. 136
      test/unit/fixture/search_linguistic_viewport_min_diagonal.js
  9. 34
      test/unit/query/search.js
  10. 25
      test/unit/sanitiser/_geo_common.js
  11. 143
      test/unit/sanitiser/search.js

10
README.md
Unescape View File

@ -9,16 +9,6 @@ See our [API Documentation](https://github.com/pelias/api/blob/master/public/api
## Install Dependencies
The API uses [elasticsearch scripts](https://github.com/pelias/scripts) for additional scoring/sorting logic. You
**must** install them, as documented [here](https://github.com/pelias/scripts#pelias-scripts). Failure to do so will
result in the following error:
```
ElasticsearchIllegalArgumentException[Unable to find on disk script admin_boost]
```
Once you are done with installing the scripts, Run the following
```bash
npm install
```

33
controller/place.js
Unescape View File

@ -13,14 +13,35 @@ function setup( backend ){
return next();
}
var query = req.clean.ids.map( function(id) {
/* req.clean.ids contains an array of objects with id and types properties.
* types is an array of one or more types, since it can't always be known which single
* type a gid might belong to (osmnode and osmway both have source osm and layer venue).
*
* However, the mget Elasticsearch query only accepts a single type at a
* time.
*
* So, first create a new array that, has an entry
* with each type and id combination. This requires creating a new array with more entries
* than req.clean.ids in the case where entries have multiple types.
*/
var recordsToReturn = req.clean.ids.reduce(function (acc, ids_element) {
ids_element.types.forEach(function(type) {
acc.push({
id: ids_element.id,
type: type
});
});
return acc;
}, []);
/*
* Next, map the list of records to an Elasticsearch mget query
*/
var query = recordsToReturn.map( function(id) {
return {
_index: 'pelias',
/*
* some gids aren't resolvable to a single type (ex: osmnode and osmway
* both have source osm and layer venue), so expect an array of
* possible values. */
_type: id.types,
_type: id.type,
_id: id.id
};
});

29
query/search.js
Unescape View File

@ -1,7 +1,8 @@
var peliasQuery = require('pelias-query'),
defaults = require('./defaults'),
textParser = require('./text_parser'),
check = require('check-types');
check = require('check-types'),
geolib = require('geolib');
//------------------------------
// general-purpose search query
@ -65,13 +66,15 @@ function generateQuery( clean ){
}
// focus viewport
// @todo: change these to the correct request variable names
// @todo: calculate the centroid from the viewport box
if( clean.focus && clean.focus.viewport ){
var vp = clean.focus.viewport;
if( check.number(clean['focus.viewport.min_lat']) &&
check.number(clean['focus.viewport.max_lat']) &&
check.number(clean['focus.viewport.min_lon']) &&
check.number(clean['focus.viewport.max_lon']) ) {
// calculate the centroid from the viewport box
vs.set({
'focus:point:lat': vp.min_lat + ( vp.max_lat - vp.min_lat ) / 2,
'focus:point:lon': vp.min_lon + ( vp.max_lon - vp.min_lon ) / 2
'focus:point:lat': clean['focus.viewport.min_lat'] + ( clean['focus.viewport.max_lat'] - clean['focus.viewport.min_lat'] ) / 2,
'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2,
'focus:scale': calculateDiagonalDistance(clean) + 'km'
});
}
@ -119,4 +122,16 @@ function generateQuery( clean ){
return query.render( vs );
}
// return diagonal distance in km, with min=1
function calculateDiagonalDistance(clean) {
var diagonalDistance = geolib.getDistance(
{ latitude: clean['focus.viewport.min_lat'], longitude: clean['focus.viewport.min_lon'] },
{ latitude: clean['focus.viewport.max_lat'], longitude: clean['focus.viewport.max_lon'] },
1000
) / 1000;
return Math.max(diagonalDistance, 1);
}
module.exports = generateQuery;

16
sanitiser/_geo_common.js
Unescape View File

@ -36,7 +36,7 @@ function sanitize_rect( key_prefix, clean, raw, bbox_is_required ) {
// check each property individually. now that it is known a bbox is present,
// all properties must exist, so pass the true flag for coord_is_required
properties.forEach(function(prop) {
sanitize_coord(prop, clean, raw[prop], true);
sanitize_coord(prop, clean, raw, true);
});
}
@ -52,7 +52,7 @@ function sanitize_circle( key_prefix, clean, raw, circle_is_required ) {
// sanitize both a point and a radius if radius is present
// otherwise just sanittize the point
if( check.assigned( raw[ key_prefix + '.radius' ] ) ){
sanitize_coord( key_prefix + '.radius', clean, raw[ key_prefix + '.radius' ], true );
sanitize_coord( key_prefix + '.radius', clean, raw, true );
sanitize_point( key_prefix, clean, raw, true);
} else {
sanitize_point( key_prefix, clean, raw, circle_is_required);
@ -89,20 +89,20 @@ function sanitize_point( key_prefix, clean, raw, point_is_required ) {
// check each property individually. now that it is known a bbox is present,
// all properties must exist, so pass the true flag for coord_is_required
properties.forEach(function(prop) {
sanitize_coord(prop, clean, raw[prop], true);
sanitize_coord(prop, clean, raw, true);
});
}
/**
* Validate lat,lon values
*
* @param {string} key
* @param {object} clean
* @param {string} rawValue
* @param {string} key - which key to validate
* @param {object} clean - cleaned parameters object
* @param {object} raw - the raw request object
* @param {bool} latlon_is_required
*/
function sanitize_coord( key, clean, rawValue, latlon_is_required ) {
var parsedValue = parseFloat( rawValue );
function sanitize_coord( key, clean, raw, latlon_is_required ) {
var parsedValue = parseFloat( raw[key] );
if ( _.isFinite( parsedValue ) ) {
clean[key] = parsedValue;
}

14
sanitiser/_geo_search.js
Unescape View File

@ -1,7 +1,9 @@
var check = require('check-types');
var geo_common = require ('./_geo_common');
var LAT_LON_IS_REQUIRED = false;
var RECT_IS_REQUIRED = false;
var CIRCLE_IS_REQUIRED = false;
var VIEWPORT_IS_REQUIRED = false;
// validate inputs, convert types and apply defaults
module.exports = function sanitize( raw, clean ){
@ -9,10 +11,22 @@ module.exports = function sanitize( raw, clean ){
// error & warning messages
var messages = { errors: [], warnings: [] };
// disallow specifying both focus.point and focus.viewport
if ( ( raw['focus.viewport.min_lat'] ||
raw['focus.viewport.max_lat'] ||
raw['focus.viewport.min_lon'] ||
raw['focus.viewport.max_lon'] ) &&
( raw['focus.point.lat'] ||
raw['focus.point.lon'] ) ) {
messages.errors.push( 'focus.point and focus.viewport can\'t both be set' );
return messages;
}
try {
geo_common.sanitize_point( 'focus.point', clean, raw, LAT_LON_IS_REQUIRED );
geo_common.sanitize_rect( 'boundary.rect', clean, raw, RECT_IS_REQUIRED );
geo_common.sanitize_circle( 'boundary.circle', clean, raw, CIRCLE_IS_REQUIRED );
geo_common.sanitize_rect( 'focus.viewport', clean, raw, VIEWPORT_IS_REQUIRED );
}
catch (err) {
messages.errors.push( err.message );

4
test/unit/controller/place.js
Unescape View File

@ -41,7 +41,7 @@ module.exports.tests.functional_success = function(test, common) {
test('functional success', function(t) {
var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command');
t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
});
var controller = setup( backend );
var res = {
@ -70,7 +70,7 @@ module.exports.tests.functional_success = function(test, common) {
module.exports.tests.functional_failure = function(test, common) {
test('functional failure', function(t) {
var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'b' ] } ] } }, 'correct backend command');
t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command');
});
var controller = setup( backend );
var req = { clean: { ids: [ {'id' : 123, types: [ 'b' ] } ] }, errors: [], warnings: [] };

136
test/unit/fixture/search_linguistic_viewport.js
Unescape View File

@ -0,0 +1,136 @@
module.exports = {
'query': {
'filtered': {
'query': {
'bool': {
'must': [
{
'match': {
'name.default': {
'analyzer': 'peliasOneEdgeGram',
'boost': 1,
'query': 'test'
}
}
}
],
'should': [
{
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'functions': [
{
'weight': 2,
'linear': {
'center_point': {
'origin': {
'lat': 29.49136,
'lon': -82.50622
},
'offset': '1km',
'scale': '994km',
'decay': 0.5
}
}
}
],
'score_mode': 'avg',
'boost_mode': 'replace'
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'max_boost': 20,
'functions': [
{
'field_value_factor': {
'modifier': 'log1p',
'field': 'popularity'
},
'weight': 1
}
],
'score_mode': 'first',
'boost_mode': 'replace',
'filter': {
'exists': {
'field': 'popularity'
}
}
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'max_boost': 20,
'functions': [
{
'field_value_factor': {
'modifier': 'log1p',
'field': 'population'
},
'weight': 2
}
],
'score_mode': 'first',
'boost_mode': 'replace',
'filter': {
'exists': {
'field': 'population'
}
}
}
}
]
}
}
}
},
'size': 10,
'track_scores': true,
'sort': [
'_score'
]
};

136
test/unit/fixture/search_linguistic_viewport_min_diagonal.js
Unescape View File

@ -0,0 +1,136 @@
module.exports = {
'query': {
'filtered': {
'query': {
'bool': {
'must': [
{
'match': {
'name.default': {
'analyzer': 'peliasOneEdgeGram',
'boost': 1,
'query': 'test'
}
}
}
],
'should': [
{
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'functions': [
{
'weight': 2,
'linear': {
'center_point': {
'origin': {
'lat': 28.49136,
'lon': -87.50623
},
'offset': '1km',
'scale': '1km',
'decay': 0.5
}
}
}
],
'score_mode': 'avg',
'boost_mode': 'replace'
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'max_boost': 20,
'functions': [
{
'field_value_factor': {
'modifier': 'log1p',
'field': 'popularity'
},
'weight': 1
}
],
'score_mode': 'first',
'boost_mode': 'replace',
'filter': {
'exists': {
'field': 'popularity'
}
}
}
},
{
'function_score': {
'query': {
'match': {
'phrase.default': {
'analyzer': 'peliasPhrase',
'type': 'phrase',
'boost': 1,
'slop': 2,
'query': 'test'
}
}
},
'max_boost': 20,
'functions': [
{
'field_value_factor': {
'modifier': 'log1p',
'field': 'population'
},
'weight': 2
}
],
'score_mode': 'first',
'boost_mode': 'replace',
'filter': {
'exists': {
'field': 'population'
}
}
}
}
]
}
}
}
},
'size': 10,
'track_scores': true,
'sort': [
'_score'
]
};

34
test/unit/query/search.js
Unescape View File

@ -73,6 +73,40 @@ module.exports.tests.query = function(test, common) {
t.end();
});
test('search search + viewport', function(t) {
var query = generate({
text: 'test', size: 10,
'focus.viewport.min_lat': 28.49136,
'focus.viewport.max_lat': 30.49136,
'focus.viewport.min_lon': -87.50622,
'focus.viewport.max_lon': -77.50622,
layers: ['test']
});
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/search_linguistic_viewport');
t.deepEqual(compiled, expected, 'valid search query');
t.end();
});
test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
var query = generate({
text: 'test', size: 10,
'focus.viewport.min_lat': 28.49135,
'focus.viewport.max_lat': 28.49137,
'focus.viewport.min_lon': -87.50622,
'focus.viewport.max_lon': -87.50624,
layers: ['test']
});
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
t.deepEqual(compiled, expected, 'valid search query');
t.end();
});
test('search search + focus on null island', function(t) {
var query = generate({
text: 'test', size: 10,

25
test/unit/sanitiser/_geo_common.js
Unescape View File

@ -20,7 +20,7 @@ module.exports.tests.coord = function(test, common) {
};
var mandatory = false;
sanitize.sanitize_coord( 'foo', clean, params.foo, mandatory );
sanitize.sanitize_coord( 'foo', clean, params, mandatory );
t.equal(clean.foo, params.foo);
t.end();
});
@ -32,7 +32,7 @@ module.exports.tests.coord = function(test, common) {
};
var mandatory = false;
sanitize.sanitize_coord( 'foo', clean, params.foo, mandatory );
sanitize.sanitize_coord( 'foo', clean, params, mandatory );
t.equal(clean.foo, undefined, 'not set');
t.end();
});
@ -43,7 +43,7 @@ module.exports.tests.coord = function(test, common) {
var mandatory = false;
t.doesNotThrow( function(){
sanitize.sanitize_coord( 'foo', clean, params.foo, mandatory );
sanitize.sanitize_coord( 'foo', clean, params, mandatory );
});
t.end();
});
@ -54,7 +54,7 @@ module.exports.tests.coord = function(test, common) {
var mandatory = true;
t.throws( function(){
sanitize.sanitize_coord( 'foo', clean, params.foo, mandatory );
sanitize.sanitize_coord( 'foo', clean, params, mandatory );
});
t.end();
});
@ -172,6 +172,23 @@ module.exports.tests.rect = function(test, common) {
t.end();
});
test('valid rect quad, null island', function (t) {
var clean = {};
var params = {
'boundary.rect.min_lat': 0,
'boundary.rect.max_lat': 0,
'boundary.rect.min_lon': 0,
'boundary.rect.max_lon': 0
};
var mandatory = false;
sanitize.sanitize_rect( 'boundary.rect', clean, params, mandatory );
t.equal(clean['boundary.rect.min_lat'], params['boundary.rect.min_lat'], 'min_lat approved');
t.equal(clean['boundary.rect.max_lat'], params['boundary.rect.max_lat'], 'min_lat approved');
t.equal(clean['boundary.rect.min_lon'], params['boundary.rect.min_lon'], 'min_lat approved');
t.equal(clean['boundary.rect.max_lon'], params['boundary.rect.max_lon'], 'min_lat approved');
t.end();
});
test('invalid rect - partially specified', function (t) {
var clean = {};
var params = {

143
test/unit/sanitiser/search.js
Unescape View File

@ -5,7 +5,6 @@ var extend = require('extend'),
middleware = search.middleware,
defaultError = 'invalid param \'text\': text length, must be >0';
// these are the default values you would expect when no input params are specified.
// @todo: why is this different from $defaultClean?
var emptyClean = { private: false, size: 10, types: {} };
module.exports.tests = {};
@ -186,74 +185,94 @@ module.exports.tests.sanitize_optional_geo = function(test, common) {
};
module.exports.tests.sanitize_bounding_rect = function(test, common) {
test('valid bounding rect', function(t) {
var req = {
query: {
text: 'test',
'boundary.rect.min_lat': -40.659,
'boundary.rect.max_lat': -41.614,
'boundary.rect.min_lon': 174.612,
'boundary.rect.max_lon': 176.333
}
};
// convernience function to avoid refactoring the succict geojson bbox
// fixtures in to the more verbose bounding.rect format.
var mapGeoJsonBboxFormatToBoundingRectFormat = function( bbox ){
var split = bbox.split(',');
return {
'boundary.rect.min_lon': split[0],
'boundary.rect.max_lat': split[1],
'boundary.rect.max_lon': split[2],
'boundary.rect.min_lat': split[3]
sanitize(req, function(){
t.equal(req.errors[0], undefined, 'no error');
t.equal(req.clean['boundary.rect.min_lon'], parseFloat(req.query['boundary.rect.min_lon']));
t.equal(req.clean['boundary.rect.max_lat'], parseFloat(req.query['boundary.rect.max_lat']));
t.equal(req.clean['boundary.rect.max_lon'], parseFloat(req.query['boundary.rect.max_lon']));
t.equal(req.clean['boundary.rect.min_lat'], parseFloat(req.query['boundary.rect.min_lat']));
t.end();
});
});
};
module.exports.tests.sanitize_viewport = function(test, common) {
test('valid viewport', function(t) {
var req = {
query: {
text: 'test',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lat': '38',
'focus.viewport.min_lon': '-123',
'focus.viewport.max_lon': '-122'
}
};
};
sanitize(req, function() {
t.equal(req.errors[0], undefined, 'no error');
t.equal(req.clean['focus.viewport.min_lat'], parseFloat(req.query['focus.viewport.min_lat']), 'correct min_lat in clean');
t.equal(req.clean['focus.viewport.max_lat'], parseFloat(req.query['focus.viewport.max_lat']), 'correct max_lat in clean');
t.equal(req.clean['focus.viewport.min_lon'], parseFloat(req.query['focus.viewport.min_lon']), 'correct min_lon in clean');
t.equal(req.clean['focus.viewport.max_lon'], parseFloat(req.query['focus.viewport.max_lon']), 'correct max_lon in clean');
t.end();
});
});
var bboxes = {
invalid: [
'91;-181,-91,181', // invalid - semicolon between coordinates
'these,are,not,numbers',
'0,0,0,notANumber',
',,,',
'91, -181, -91', // invalid - missing a coordinate
'123,12', // invalid - missing coordinates
'' // invalid - empty param
].map(mapGeoJsonBboxFormatToBoundingRectFormat),
valid: [
'-179,90,34,-80', // valid top_right lon/lat, bottom_left lon/lat
'0,0,0,0',
'10,20,30,40',
'-40,-20,10,30',
'-40,-20,10,30',
'-1200,20,1000,20',
'-1400,90,1400,-90',
// wrapped latitude coordinates
'-181,90,34,-180',
'-170,91,-181,45',
'-181,91,181,-91',
'91, -181,-91,11',
'91, -11,-91,181'
].map(mapGeoJsonBboxFormatToBoundingRectFormat)
};
test('error returned if focus.point and focus.viewpoint specified', function(t) {
var req = {
query: {
text: 'test',
'focus.point.lat': '10',
'focus.point.lon': '15',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lat': '38',
'focus.viewport.min_lon': '-123',
'focus.viewport.max_lon': '-122'
}
};
test('invalid bounding rect', function(t) {
bboxes.invalid.forEach( function( bbox ){
var req = { query: { text: 'test' } };
extend( req.query, bbox );
sanitize(req, function(){
t.equal(req.clean['boundary.rect.min_lon'], undefined);
t.equal(req.clean['boundary.rect.max_lat'], undefined);
t.equal(req.clean['boundary.rect.max_lon'], undefined);
t.equal(req.clean['boundary.rect.min_lat'], undefined);
t.equal(req.errors.length, 1, 'bounding error');
});
sanitize(req, function() {
t.equal(req.errors[0], 'focus.point and focus.viewport can\'t both be set', 'no error');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lon'), 'clean should be empty');
t.end();
});
t.end();
});
test('valid bounding rect', function(t) {
bboxes.valid.forEach( function( bbox ){
var req = { query: { text: 'test' } };
extend( req.query, bbox );
sanitize(req, function(){
t.equal(req.errors[0], undefined, 'no error');
t.equal(req.clean['boundary.rect.min_lon'], parseFloat(bbox['boundary.rect.min_lon']));
t.equal(req.clean['boundary.rect.max_lat'], parseFloat(bbox['boundary.rect.max_lat']));
t.equal(req.clean['boundary.rect.max_lon'], parseFloat(bbox['boundary.rect.max_lon']));
t.equal(req.clean['boundary.rect.min_lat'], parseFloat(bbox['boundary.rect.min_lat']));
});
test('error returned if focus.point and focus.viewpoint partially specified', function(t) {
var req = {
query: {
text: 'test',
'focus.point.lat': '10',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lon': '-122'
}
};
sanitize(req, function() {
t.equal(req.errors[0], 'focus.point and focus.viewport can\'t both be set', 'no error');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lon'), 'clean should be empty');
t.end();
});
t.end();
});
};

Write Preview
Loading…
Cancel
Save