Browse Source

Merge pull request #626 from pelias/staging

Merge staging into production
pull/745/head
Peter Johnson 8 years ago committed by GitHub
parent
commit
de897afef9
  1. 8
      .travis.yml
  2. 2
      README.md
  3. 2
      helper/type_mapping.js
  4. 14
      package.json
  5. 29
      query/search.js
  6. 8
      sanitiser/_geo_common.js
  7. 14
      sanitiser/_geo_search.js
  8. 44
      sanitiser/wrap.js
  9. 2
      test/unit/fixture/search_partial_address.js
  10. 36
      test/unit/query/search.js
  11. 1
      test/unit/run.js
  12. 59
      test/unit/sanitiser/_geo_common.js
  13. 11
      test/unit/sanitiser/_sources_and_layers.js
  14. 2
      test/unit/sanitiser/reverse.js
  15. 71
      test/unit/sanitiser/search.js
  16. 191
      test/unit/sanitiser/wrap.js

8
.travis.yml

@ -28,3 +28,11 @@ before_install:
- npm i -g npm@^2.0.0
before_script:
- npm prune
after_success:
- 'curl -Lo travis_after_all.py https://git.io/travis_after_all'
- python travis_after_all.py
- export $(cat .to_export_back) &> /dev/null
- npm run semantic-release
branches:
except:
- /^v\d+\.\d+\.\d+$/

2
README.md

@ -7,7 +7,7 @@
This is the API server for the Pelias project. It's the service that runs to process user HTTP requests and return results as GeoJSON by querying Elasticsearch.
[![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/pelias/api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pelias/api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Build Status](https://travis-ci.org/pelias/api.png?branch=master)](https://travis-ci.org/pelias/api)
## Documentation

2
helper/type_mapping.js

@ -48,7 +48,7 @@ var LAYERS_BY_SOURCE = {
openstreetmap: [ 'address', 'venue' ],
openaddresses: [ 'address' ],
geonames: [ 'country','macroregion', 'region', 'county','localadmin',
'locality', 'neighbourhood', 'venue' ],
'locality','borough', 'neighbourhood', 'venue' ],
whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region',
'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
'neighbourhood', 'microhood', 'disputed']

14
package.json

@ -1,7 +1,7 @@
{
"name": "pelias-api",
"version": "0.0.0-semantic-release",
"author": "mapzen",
"version": "2.2.0",
"description": "Pelias API",
"homepage": "https://github.com/pelias/api",
"license": "MIT",
@ -16,11 +16,12 @@
"test": "npm run unit",
"travis": "npm test",
"unit": "./bin/units",
"validate": "npm ls"
"validate": "npm ls",
"semantic-release": "semantic-release pre && npm publish && semantic-release post"
},
"repository": {
"type": "git",
"url": "git://github.com/pelias/api.git"
"url": "https://github.com/pelias/api.git"
},
"keywords": [
"pelias",
@ -52,8 +53,8 @@
"pelias-config": "2.1.0",
"pelias-logger": "0.0.8",
"pelias-model": "4.1.0",
"pelias-query": "8.1.3",
"pelias-text-analyzer": "1.2.0",
"pelias-query": "8.3.0",
"pelias-text-analyzer": "1.3.0",
"stats-lite": "2.0.3",
"through2": "2.0.1"
},
@ -68,7 +69,8 @@
"source-map": "^0.5.6",
"tap-dot": "1.0.5",
"tape": "^4.5.1",
"uglify-js": "^2.6.2"
"uglify-js": "^2.6.2",
"semantic-release": "^4.3.5"
},
"pre-commit": [
"lint",

29
query/search.js

@ -1,8 +1,8 @@
var peliasQuery = require('pelias-query'),
defaults = require('./search_defaults'),
textParser = require('./text_parser'),
check = require('check-types'),
geolib = require('geolib');
check = require('check-types');
var placeTypes = require('../helper/placeTypes');
// region_a is also an admin field. addressit tries to detect
@ -77,19 +77,6 @@ function generateQuery( clean ){
});
}
// focus viewport
if( check.number(clean['focus.viewport.min_lat']) &&
check.number(clean['focus.viewport.max_lat']) &&
check.number(clean['focus.viewport.min_lon']) &&
check.number(clean['focus.viewport.max_lon']) ) {
// calculate the centroid from the viewport box
vs.set({
'focus:point:lat': clean['focus.viewport.min_lat'] + ( clean['focus.viewport.max_lat'] - clean['focus.viewport.min_lat'] ) / 2,
'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2
//, 'focus:scale': calculateDiagonalDistance(clean) + 'km'
});
}
// boundary rect
if( check.number(clean['boundary.rect.min_lat']) &&
check.number(clean['boundary.rect.max_lat']) &&
@ -134,16 +121,4 @@ function generateQuery( clean ){
return query.render( vs );
}
// return diagonal distance in km, with min=1
function calculateDiagonalDistance(clean) {
var diagonalDistance = geolib.getDistance(
{ latitude: clean['focus.viewport.min_lat'], longitude: clean['focus.viewport.min_lon'] },
{ latitude: clean['focus.viewport.max_lat'], longitude: clean['focus.viewport.max_lon'] },
1000
) / 1000;
return Math.max(diagonalDistance, 1);
}
module.exports = generateQuery;

8
sanitiser/_geo_common.js

@ -4,6 +4,7 @@
var groups = require('./_groups'),
util = require('util'),
check = require('check-types'),
wrap = require('./wrap'),
_ = require('lodash');
/**
@ -68,6 +69,7 @@ function sanitize_circle( key_prefix, clean, raw, circle_is_required ) {
* @param {bool} point_is_required
*/
function sanitize_point( key_prefix, clean, raw, point_is_required ) {
// calculate full property names from the key_prefix
var properties = [ 'lat', 'lon'].map(function(prop) {
return key_prefix + '.' + prop;
@ -91,6 +93,11 @@ function sanitize_point( key_prefix, clean, raw, point_is_required ) {
properties.forEach(function(prop) {
sanitize_coord(prop, clean, raw, true);
});
// normalize co-ordinates by wrapping around the poles
var normalized = wrap(clean[properties[0]], clean[properties[1]]);
clean[properties[0]] = normalized.lat;
clean[properties[1]] = normalized.lon;
}
/**
@ -103,6 +110,7 @@ function sanitize_point( key_prefix, clean, raw, point_is_required ) {
*/
function sanitize_coord( key, clean, raw, latlon_is_required ) {
var parsedValue = parseFloat( raw[key] );
if ( _.isFinite( parsedValue ) ) {
clean[key] = parsedValue;
}

14
sanitiser/_geo_search.js

@ -1,9 +1,9 @@
var check = require('check-types');
var geo_common = require ('./_geo_common');
var LAT_LON_IS_REQUIRED = false;
var RECT_IS_REQUIRED = false;
var CIRCLE_IS_REQUIRED = false;
var VIEWPORT_IS_REQUIRED = false;
// validate inputs, convert types and apply defaults
module.exports = function sanitize( raw, clean ){
@ -11,22 +11,10 @@ module.exports = function sanitize( raw, clean ){
// error & warning messages
var messages = { errors: [], warnings: [] };
// disallow specifying both focus.point and focus.viewport
if ( ( raw['focus.viewport.min_lat'] ||
raw['focus.viewport.max_lat'] ||
raw['focus.viewport.min_lon'] ||
raw['focus.viewport.max_lon'] ) &&
( raw['focus.point.lat'] ||
raw['focus.point.lon'] ) ) {
messages.errors.push( 'focus.point and focus.viewport can\'t both be set' );
return messages;
}
try {
geo_common.sanitize_point( 'focus.point', clean, raw, LAT_LON_IS_REQUIRED );
geo_common.sanitize_rect( 'boundary.rect', clean, raw, RECT_IS_REQUIRED );
geo_common.sanitize_circle( 'boundary.circle', clean, raw, CIRCLE_IS_REQUIRED );
geo_common.sanitize_rect( 'focus.viewport', clean, raw, VIEWPORT_IS_REQUIRED );
}
catch (err) {
messages.errors.push( err.message );

44
sanitiser/wrap.js

@ -0,0 +1,44 @@
/**
normalize co-ordinates that lie outside of the normal ranges.
longitude wrapping simply requires adding +- 360 to the value until it comes
in to range.
for the latitude values we need to flip the longitude whenever the latitude
crosses a pole.
**/
function wrap( lat, lon ){
var point = { lat: lat, lon: lon };
var quadrant = Math.floor( Math.abs(lat) / 90) % 4;
var pole = ( lat > 0 ) ? 90 : -90;
var offset = lat % 90;
switch( quadrant ){
case 0:
point.lat = offset;
break;
case 1:
point.lat = pole - offset;
point.lon += 180;
break;
case 2:
point.lat = -offset;
point.lon += 180;
break;
case 3:
point.lat = -pole + offset;
break;
}
if( point.lon > 180 || point.lon <= -180 ){
point.lon -= Math.floor(( point.lon + 180 ) / 360) * 360;
}
return point;
}
module.exports = wrap;

2
test/unit/fixture/search_partial_address.js

@ -78,7 +78,7 @@ module.exports = {
'parent.region_a': {
'analyzer': 'peliasAdmin',
'boost': 1,
'query': 'new york'
'query': 'NY'
}
}
}, {

36
test/unit/query/search.js

@ -73,42 +73,6 @@ module.exports.tests.query = function(test, common) {
t.end();
});
test('search search + viewport', function(t) {
var query = generate({
text: 'test', querySize: 10,
'focus.viewport.min_lat': 28.49136,
'focus.viewport.max_lat': 30.49136,
'focus.viewport.min_lon': -87.50622,
'focus.viewport.max_lon': -77.50622,
layers: ['test']
});
var compiled = JSON.parse( JSON.stringify( query ) );
var expected = require('../fixture/search_linguistic_viewport');
t.deepEqual(compiled, expected, 'search_linguistic_viewport');
t.end();
});
// viewport scale sizing currently disabled.
// ref: https://github.com/pelias/api/pull/388
// test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
// var query = generate({
// text: 'test', querySize: 10,
// 'focus.viewport.min_lat': 28.49135,
// 'focus.viewport.max_lat': 28.49137,
// 'focus.viewport.min_lon': -87.50622,
// 'focus.viewport.max_lon': -87.50624,
// layers: ['test']
// });
//
// var compiled = JSON.parse( JSON.stringify( query ) );
// var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
//
// t.deepEqual(compiled, expected, 'valid search query');
// t.end();
// });
test('search search + focus on null island', function(t) {
var query = generate({
text: 'test', querySize: 10,

1
test/unit/run.js

@ -56,6 +56,7 @@ var tests = [
require('./sanitiser/place'),
require('./sanitiser/reverse'),
require('./sanitiser/search'),
require('./sanitiser/wrap'),
require('./service/mget'),
require('./service/search'),
];

59
test/unit/sanitiser/_geo_common.js

@ -12,6 +12,65 @@ module.exports.tests.interface = function(test, common) {
});
};
// @note: for better coverage see unit tests for 'wrap.js'.
module.exports.tests.wrapping = function(test, common) {
test('control - no wrapping required', function (t) {
var clean = {};
var params = {
'point.lat': +1.1,
'point.lon': -1.1
};
sanitize.sanitize_point( 'point', clean, params, false );
t.equal(clean['point.lat'], +1.1, 'not changed');
t.equal(clean['point.lon'], -1.1, 'not changed');
t.end();
});
test('positive longitude wrapping', function (t) {
var clean = {};
var params = {
'point.lat': +1.1,
'point.lon': +181.1
};
sanitize.sanitize_point( 'point', clean, params, false );
t.equal(clean['point.lat'], +1.1, 'not changed');
t.equal(clean['point.lon'], -178.9, 'equal to (-180 + 1.1)');
t.end();
});
test('negative longitude wrapping', function (t) {
var clean = {};
var params = {
'point.lat': -1.1,
'point.lon': -181.1
};
sanitize.sanitize_point( 'point', clean, params, false );
t.equal(clean['point.lat'], -1.1, 'not changed');
t.equal(clean['point.lon'], +178.9, 'equal to (+180 - 1.1)');
t.end();
});
test('positive latitudinal wrapping', function (t) {
var clean = {};
var params = {
'point.lat': 91.1,
'point.lon': 1.1
};
sanitize.sanitize_point( 'point', clean, params, false );
t.equal(clean['point.lat'], +88.9, 'equal to (+90 - 1.1)');
t.equal(clean['point.lon'], -178.9, 'equal to (-180 + 1.1)'); // polar flip
t.end();
});
test('negative latitudinal wrapping', function (t) {
var clean = {};
var params = {
'point.lat': -91.1,
'point.lon': -1.1
};
sanitize.sanitize_point( 'point', clean, params, false );
t.equal(clean['point.lat'], -88.9, 'equal to (-90 + 1.1)');
t.equal(clean['point.lon'], +178.9, 'equal to (+180 - 1.1)'); // polar flip
t.end();
});
};
module.exports.tests.coord = function(test, common) {
test('valid coord', function (t) {
var clean = {};

11
test/unit/sanitiser/_sources_and_layers.js

@ -51,6 +51,17 @@ module.exports.tests.no_errors = function(test, common) {
t.end();
});
test('valid combination', function(t) {
var raw = {};
var clean = { sources: ['geonames'], layers: ['borough'] };
var messages = sanitize(raw, clean);
t.equal(messages.errors.length, 0, 'should return no errors');
t.equal(messages.warnings.length, 0, 'should return no warnings');
t.end();
});
test('valid combination', function(t) {
var raw = {};
var clean = { sources: ['geonames'], layers: ['macroregion'] };

2
test/unit/sanitiser/reverse.js

@ -65,7 +65,6 @@ module.exports.tests.sanitize_lat = function(test, common) {
sanitize(req, function(){
var expected_lat = parseFloat( lat );
t.deepEqual(req.errors, [], 'no errors');
t.equal(req.clean['point.lat'], expected_lat, 'clean set correctly (' + lat + ')');
});
});
t.end();
@ -93,7 +92,6 @@ module.exports.tests.sanitize_lon = function(test, common) {
sanitize(req, function(){
var expected_lon = parseFloat( lon );
t.deepEqual(req.errors, [], 'no errors');
t.equal(req.clean['point.lon'], expected_lon, 'clean set correctly (' + lon + ')');
});
});
t.end();

71
test/unit/sanitiser/search.js

@ -129,7 +129,6 @@ module.exports.tests.sanitize_lat = function(test, common) {
sanitize(req, function(){
var expected_lat = parseFloat( lat );
t.equal(req.errors[0], undefined, 'no error');
t.equal(req.clean['focus.point.lat'], expected_lat, 'clean lat set correctly (' + lat + ')');
});
});
t.end();
@ -146,7 +145,6 @@ module.exports.tests.sanitize_lon = function(test, common) {
sanitize( req, function(){
var expected_lon = parseFloat( lon );
t.equal(req.errors[0], undefined, 'no error');
t.deepEqual(req.clean['focus.point.lon'], expected_lon, 'clean set correctly (' + lon + ')');
});
});
t.end();
@ -208,75 +206,6 @@ module.exports.tests.sanitize_bounding_rect = function(test, common) {
});
};
module.exports.tests.sanitize_viewport = function(test, common) {
test('valid viewport', function(t) {
var req = {
query: {
text: 'test',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lat': '38',
'focus.viewport.min_lon': '-123',
'focus.viewport.max_lon': '-122'
}
};
sanitize(req, function() {
t.equal(req.errors[0], undefined, 'no error');
t.equal(req.clean['focus.viewport.min_lat'], parseFloat(req.query['focus.viewport.min_lat']), 'correct min_lat in clean');
t.equal(req.clean['focus.viewport.max_lat'], parseFloat(req.query['focus.viewport.max_lat']), 'correct max_lat in clean');
t.equal(req.clean['focus.viewport.min_lon'], parseFloat(req.query['focus.viewport.min_lon']), 'correct min_lon in clean');
t.equal(req.clean['focus.viewport.max_lon'], parseFloat(req.query['focus.viewport.max_lon']), 'correct max_lon in clean');
t.end();
});
});
test('error returned if focus.point and focus.viewpoint specified', function(t) {
var req = {
query: {
text: 'test',
'focus.point.lat': '10',
'focus.point.lon': '15',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lat': '38',
'focus.viewport.min_lon': '-123',
'focus.viewport.max_lon': '-122'
}
};
sanitize(req, function() {
t.equal(req.errors[0], 'focus.point and focus.viewport can\'t both be set', 'no error');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lon'), 'clean should be empty');
t.end();
});
});
test('error returned if focus.point and focus.viewpoint partially specified', function(t) {
var req = {
query: {
text: 'test',
'focus.point.lat': '10',
'focus.viewport.min_lat': '37',
'focus.viewport.max_lon': '-122'
}
};
sanitize(req, function() {
t.equal(req.errors[0], 'focus.point and focus.viewport can\'t both be set', 'no error');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.min_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.viewport.max_lon'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lat'), 'clean should be empty');
t.notOk(req.clean.hasOwnProperty('focus.point.lon'), 'clean should be empty');
t.end();
});
});
};
module.exports.tests.sanitize_size = function(test, common) {
test('invalid size value', function(t) {
var req = { query: { size: 'a', text: 'test', lat: 0, lon: 0 } };

191
test/unit/sanitiser/wrap.js

@ -0,0 +1,191 @@
var wrap = require('../../../sanitiser/wrap');
module.exports.tests = {};
module.exports.tests.control = function(test, common) {
test('control - no wrapping required', function (t) {
var norm = wrap(55.555, 22.222);
t.equal(norm.lat, 55.555);
t.equal(norm.lon, 22.222);
t.end();
});
};
module.exports.tests.latitude_positive = function(test, common) {
test('positive latitude wrapping - 1 degree', function (t) {
var norm = wrap(1, 0);
t.equal(norm.lat, 1);
t.equal(norm.lon, 0);
t.end();
});
test('positive latitude wrapping - 91 degrees', function (t) {
var norm = wrap(91, 0);
t.equal(norm.lat, 89);
t.equal(norm.lon, 180);
t.end();
});
test('positive latitude wrapping - 181 degrees', function (t) {
var norm = wrap(181, 0);
t.equal(norm.lat, -1);
t.equal(norm.lon, 180);
t.end();
});
test('positive latitude wrapping - 271 degrees', function (t) {
var norm = wrap(271, 0);
t.equal(norm.lat, -89);
t.equal(norm.lon, 0);
t.end();
});
test('positive latitude wrapping - 361 degrees', function (t) {
var norm = wrap(361, 0);
t.equal(norm.lat, 1);
t.equal(norm.lon, 0);
t.end();
});
test('positive latitude wrapping - 631 degrees', function (t) {
var norm = wrap(631, 0);
t.equal(norm.lat, -89);
t.equal(norm.lon, 0);
t.end();
});
test('positive latitude wrapping - 721 degrees', function (t) {
var norm = wrap(721, 0);
t.equal(norm.lat, 1);
t.equal(norm.lon, 0);
t.end();
});
};
module.exports.tests.latitude_negative = function(test, common) {
test('negative latitude wrapping - 1 degree', function (t) {
var norm = wrap(-1, 0);
t.equal(norm.lat, -1);
t.equal(norm.lon, 0);
t.end();
});
test('negative latitude wrapping - 91 degrees', function (t) {
var norm = wrap(-91, 0);
t.equal(norm.lat, -89);
t.equal(norm.lon, 180);
t.end();
});
test('negative latitude wrapping - 181 degrees', function (t) {
var norm = wrap(-181, 0);
t.equal(norm.lat, 1);
t.equal(norm.lon, 180);
t.end();
});
test('negative latitude wrapping - 271 degrees', function (t) {
var norm = wrap(-271, 0);
t.equal(norm.lat, 89);
t.equal(norm.lon, 0);
t.end();
});
test('negative latitude wrapping - 361 degrees', function (t) {
var norm = wrap(-361, 0);
t.equal(norm.lat, -1);
t.equal(norm.lon, 0);
t.end();
});
test('negative latitude wrapping - 631 degrees', function (t) {
var norm = wrap(-631, 0);
t.equal(norm.lat, 89);
t.equal(norm.lon, 0);
t.end();
});
test('positive latitude wrapping - 721 degrees', function (t) {
var norm = wrap(721, 0);
t.equal(norm.lat, 1);
t.equal(norm.lon, 0);
t.end();
});
};
module.exports.tests.longitude_positive = function(test, common) {
test('positive longitude wrapping - 1 degree', function (t) {
var norm = wrap(0, 1);
t.equal(norm.lat, 0);
t.equal(norm.lon, 1);
t.end();
});
test('positive longitude wrapping - 181 degrees', function (t) {
var norm = wrap(0, 181);
t.equal(norm.lat, 0);
t.equal(norm.lon, -179);
t.end();
});
test('positive longitude wrapping - 271 degrees', function (t) {
var norm = wrap(0, 271);
t.equal(norm.lat, 0);
t.equal(norm.lon, -89);
t.end();
});
test('positive longitude wrapping - 361 degrees', function (t) {
var norm = wrap(0, 361);
t.equal(norm.lat, 0);
t.equal(norm.lon, 1);
t.end();
});
test('positive longitude wrapping - 631 degrees', function (t) {
var norm = wrap(0, 631);
t.equal(norm.lat, 0);
t.equal(norm.lon, -89);
t.end();
});
test('positive longitude wrapping - 721 degrees', function (t) {
var norm = wrap(0, 721);
t.equal(norm.lat, 0);
t.equal(norm.lon, 1);
t.end();
});
};
module.exports.tests.longitude_negative = function(test, common) {
test('negative longitude wrapping - 1 degree', function (t) {
var norm = wrap(0, -1);
t.equal(norm.lat, 0);
t.equal(norm.lon, -1);
t.end();
});
test('negative longitude wrapping - 181 degrees', function (t) {
var norm = wrap(0, -181);
t.equal(norm.lat, 0);
t.equal(norm.lon, 179);
t.end();
});
test('negative longitude wrapping - 271 degrees', function (t) {
var norm = wrap(0, -271);
t.equal(norm.lat, 0);
t.equal(norm.lon, 89);
t.end();
});
test('negative longitude wrapping - 361 degrees', function (t) {
var norm = wrap(0, -361);
t.equal(norm.lat, 0);
t.equal(norm.lon, -1);
t.end();
});
test('negative longitude wrapping - 631 degrees', function (t) {
var norm = wrap(0, -631);
t.equal(norm.lat, 0);
t.equal(norm.lon, 89);
t.end();
});
test('negative longitude wrapping - 721 degrees', function (t) {
var norm = wrap(0, -721);
t.equal(norm.lat, 0);
t.equal(norm.lon, -1);
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANTIZE wrap ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};
Loading…
Cancel
Save