Browse Source

Merge branch 'source_parameter'

pull/222/head
Julian Simioni 9 years ago
parent
commit
6e8a54d500
  1. 10
      controller/search.js
  2. 2
      helper/layers.js
  3. 73
      helper/query_parser.js
  4. 41
      helper/types.js
  5. 27
      middleware/_types.js
  6. 10
      query/sources.js
  7. 4
      query/types.js
  8. 6
      routes/v1.js
  9. 18
      sanitiser/_id.js
  10. 18
      sanitiser/_input.js
  11. 19
      sanitiser/_layers.js
  12. 44
      sanitiser/_source.js
  13. 1
      sanitiser/search.js
  14. 55
      test/unit/helper/query_parser.js
  15. 96
      test/unit/helper/types.js
  16. 9
      test/unit/query/search.js
  17. 10
      test/unit/query/types.js
  18. 4
      test/unit/run.js
  19. 6
      test/unit/sanitiser/_input.js
  20. 114
      test/unit/sanitiser/_source.js
  21. 4
      test/unit/sanitiser/place.js
  22. 23
      test/unit/sanitiser/reverse.js
  23. 117
      test/unit/sanitiser/search.js
  24. 3
      test/unit/sanitiser/suggest.js

10
controller/search.js

@ -1,4 +1,3 @@
var service = { search: require('../service/search') };
function setup( backend, query ){
@ -16,13 +15,8 @@ function setup( backend, query ){
body: query( req.clean )
};
if (req.clean.layers) {
cmd.type = req.clean.layers;
}
// set type if input suggests targeting a layer(s)
if (req.clean.default_layers_set && req.clean.parsed_input) {
cmd.type = req.clean.parsed_input.target_layer || cmd.type;
if ( req.clean.hasOwnProperty('type') ) {
cmd.type = req.clean.type;
}
// query backend

2
helper/layers.js

@ -1,6 +1,6 @@
module.exports = function(alias_layers) {
// make a copy of the array so, you are not modifying original ref
var layers = alias_layers.slice(0);
var layers = alias_layers.slice(0);
// expand aliases
var expand_aliases = function(alias, layers, layer_indeces) {

73
helper/query_parser.js

@ -1,11 +1,26 @@
var parser = require('addressit');
var extend = require('extend');
var get_layers = require('../helper/layers');
var get_layers_helper = require('../helper/layers');
var delim = ',';
module.exports = function(query) {
module.exports = {};
module.exports.get_layers = function get_layers(query) {
var tokenized = query.split(/[ ,]+/);
var hasNumber = /\d/.test(query);
if (query.length <= 3 ) {
// no address parsing required
return get_layers_helper(['admin']);
} else if (tokenized.length === 1 || (tokenized.length < 3 && !hasNumber)) {
// no need to hit address layers if there's only one (or two) token(s)
return get_layers_helper(['admin', 'poi']);
}
};
module.exports.get_parsed_address = function get_parsed_address(query) {
var tokenized = query.split(/[ ,]+/);
var hasNumber = /\d/.test(query);
@ -17,42 +32,23 @@ module.exports = function(query) {
if ( delimIndex !== -1 ) {
address.name = query.substring(0, delimIndex);
address.admin_parts = query.substring(delimIndex + 1).trim();
}
}
return address;
};
var getTargetLayersWhenAddressParsingIsNotNecessary = function(query) {
var address = {};
// set target_layer if input length <= 3 characters
if (query.length <= 3 ) {
// no address parsing required
address.target_layer = get_layers(['admin']);
} else if (tokenized.length === 1 || (tokenized.length < 3 && !hasNumber)) {
// no need to hit address layers if there's only one (or two) token(s)
address.target_layer = get_layers(['admin', 'poi']);
}
return address.target_layer ? address : null;
};
var getAddressParts = function(query) {
// address parsing
var address = parser( query );
// set target_layer if input suggests no address
if (address.text === address.regions.join(' ') && !hasNumber) {
address.target_layer = get_layers(['admin', 'poi']);
// perform full address parsing
// except on queries so short they obviously can't contain an address
if (query.length > 3) {
return parser( query );
}
return address;
};
var addressWithAdminParts = getAdminPartsBySplittingOnDelim(query);
var addressWithTargetLayers= getTargetLayersWhenAddressParsingIsNotNecessary(query);
var addressWithAddressParts= !addressWithTargetLayers ? getAddressParts(query) : {};
var addressWithAddressParts= getAddressParts(query);
var parsedAddress = extend(addressWithAdminParts,
addressWithTargetLayers,
var parsedAddress = extend(addressWithAdminParts,
addressWithAddressParts);
var address_parts = [ 'name',
@ -63,13 +59,12 @@ module.exports = function(query) {
'country',
'postalcode',
'regions',
'admin_parts',
'target_layer'
'admin_parts'
];
var parsed_input = {};
address_parts.forEach(function(part){
address_parts.forEach(function(part){
if (parsedAddress[part]) {
parsed_input[part] = parsedAddress[part];
}
@ -77,17 +72,3 @@ module.exports = function(query) {
return parsed_input;
};
// parsed_input = {
// name : parsedAddress.name,
// number : parsedAddress.number,
// street : parsedAddress.street,
// city : parsedAddress.city,
// state : parsedAddress.state,
// country: parsedAddress.country,
// postalcode : parsedAddress.postalcode,
// regions: parsedAddress.regions,
// admin_parts: parsedAddress.admin_parts,
// target_layer: parsedAddress.target_layer
// }

41
helper/types.js

@ -0,0 +1,41 @@
var valid_types = require( '../query/types' );
/**
* Calculate the set-style intersection of two arrays
*/
var intersection = function intersection(set1, set2) {
return set2.filter(function(value) {
return set1.indexOf(value) !== -1;
});
};
module.exports = function calculate_types(clean_types) {
if (!clean_types) {
return undefined;
}
/* the layers and source parameters are cumulative:
* perform a set insersection of their specified types
*/
if (clean_types.from_layers || clean_types.from_source) {
var types = valid_types;
if (clean_types.from_layers) {
types = intersection(types, clean_types.from_layers);
}
if (clean_types.from_source) {
types = intersection(types, clean_types.from_source);
}
return types;
}
/*
* Type restrictions requested by the address parser should only be used
* if both the source and layers parameters are empty, so do this last
*/
if (clean_types.from_address_parser) {
return clean_types.from_address_parser;
}
};

27
middleware/_types.js

@ -0,0 +1,27 @@
var types_helper = require( '../helper/types' );
/**
* Validate the types specified to be searched.
*
* Elasticsearch interprets an empty array of types as "search anything" rather
* than "search nothing", so in the case of an empty array, return an error
* message instead of searching at all.
*/
function middleware(req, res, next) {
var types = types_helper(req.clean.types);
if (types !== undefined && types.length !== undefined) {
if (types.length === 0) {
var err = 'You have specified both the `source` and `layers` ' +
'parameters in a combination that will return no results.';
res.status(400); // 400 Bad Request
return next(err);
} else {
req.clean.type = types;
}
}
next();
}
module.exports = middleware;

10
query/sources.js

@ -0,0 +1,10 @@
/*
* Mapping from data sources to type values
*/
module.exports = {
'geonames': ['geoname'],
'openaddresses' : ['openaddresses'],
'quattroshapes': ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
};

4
query/indeces.js → query/types.js

@ -1,5 +1,5 @@
// querable indeces
// querable types
module.exports = [
'geoname',
@ -13,4 +13,4 @@ module.exports = [
'local_admin',
'osmaddress',
'openaddresses'
];
];

6
routes/v1.js

@ -9,6 +9,11 @@ var sanitisers = {
reverse: require('../sanitiser/reverse')
};
/** ----------------------- middleware ------------------------ **/
var middleware = {
types: require('../middleware/_types')
};
/** ----------------------- controllers ----------------------- **/
var controllers = {
@ -45,6 +50,7 @@ function addRoutes(app, peliasConfig) {
]),
search: createRouter([
sanitisers.search.middleware,
middleware.types,
controllers.search(),
postProc.confidenceScores(peliasConfig),
postProc.renamePlacenames(),

18
sanitiser/_id.js

@ -5,10 +5,9 @@ var isObject = require('is-object');
// so, both type and id are required fields.
function sanitize( req ){
req.clean = req.clean || {};
var params = req.query;
var indeces = require('../query/indeces');
var types = require('../query/types');
var delim = ':';
// ensure params is a valid object
@ -30,7 +29,7 @@ function sanitize( req ){
if( params && params.id && params.id.length ){
req.clean.ids = [];
params.ids = Array.isArray(params.id) ? params.id : [params.id];
// de-dupe
params.ids = params.ids.filter(function(item, pos) {
return params.ids.indexOf(item) === pos;
@ -38,7 +37,7 @@ function sanitize( req ){
for (var i=0; i<params.ids.length; i++) {
var thisparam = params.ids[i];
// basic format/ presence of ':'
if(thisparam.indexOf(delim) === -1) {
return errormessage(null, 'invalid: must be of the format type:id for ex: \'geoname:4163334\'');
@ -56,9 +55,9 @@ function sanitize( req ){
if('string' !== typeof type || !type.length){
return errormessage(thisparam);
}
// type text must be one of the indeces
if(indeces.indexOf(type) === -1){
return errormessage('type', type + ' is invalid. It must be one of these values - [' + indeces.join(', ') + ']');
// type text must be one of the types
if(types.indexOf(type) === -1){
return errormessage('type', type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']');
}
req.clean.ids.push({
id: id,
@ -66,10 +65,9 @@ function sanitize( req ){
});
}
}
return { 'error': false };
return { 'error': false };
}
// export function
module.exports = sanitize;
module.exports = sanitize;

18
sanitiser/_input.js

@ -1,12 +1,11 @@
var isObject = require('is-object');
var query_parse= require('../helper/query_parser');
var isObject = require('is-object');
var query_parser = require('../helper/query_parser');
// validate inputs, convert types and apply defaults
function sanitize( req ){
req.clean = req.clean || {};
var params= req.query;
// ensure the input params are a valid object
if( !isObject( params ) ){
params = {};
@ -14,20 +13,21 @@ function sanitize( req ){
// input text
if('string' !== typeof params.input || !params.input.length){
return {
return {
'error': true,
'message': 'invalid param \'input\': text length, must be >0'
};
}
req.clean.input = params.input;
req.clean.parsed_input = query_parse(params.input);
req.clean.parsed_input = query_parser.get_parsed_address(params.input);
req.clean.types = req.clean.layers || {};
req.clean.types.from_address_parsing = query_parser.get_layers(params.input);
return { 'error': false };
}
// export function
module.exports = sanitize;
module.exports = sanitize;

19
sanitiser/_layers.js

@ -1,28 +1,29 @@
var isObject = require('is-object'),
indeces = require('../query/indeces'),
types = require('../query/types'),
get_layers = require('../helper/layers');
// validate inputs, convert types and apply defaults
function sanitize( req ){
var clean = req.clean || {};
var params= req.query;
clean.types = clean.types || {};
// ensure the input params are a valid object
if( !isObject( params ) ){
params = {};
}
// default case (no layers specified in GET params)
// don't even set the from_layers key in this case
if('string' !== typeof params.layers || !params.layers.length){
params.layers = 'poi,admin,address'; // default layers
clean.default_layers_set = true;
return { 'error': false };
}
// decide which layers can be queried
var alias_layers = ['poi', 'admin', 'address'];
var alias_indeces = indeces.concat(alias_layers);
var alias_types = types.concat(alias_layers);
// parse GET params
var layers = params.layers.split(',').map( function( layer ){
@ -31,18 +32,18 @@ function sanitize( req ){
// validate layer names
for( var x=0; x<layers.length; x++ ){
if( -1 === alias_indeces.indexOf( layers[x] ) ){
if( -1 === alias_types.indexOf( layers[x] ) ){
return {
'error': true,
'message': 'invalid param \'layers\': must be one or more of ' + alias_indeces.join(',')
'message': 'invalid param \'layers\': must be one or more of ' + alias_types.join(',')
};
}
}
// pass validated params to next middleware
clean.layers = get_layers(layers);
clean.types.from_layers = get_layers(layers);
req.clean = clean;
return { 'error': false };
}

44
sanitiser/_source.js

@ -0,0 +1,44 @@
var isObject = require('is-object');
var sources_map = require( '../query/sources' );
var all_sources = Object.keys(sources_map);
function sanitize( req ) {
req.clean = req.clean || {};
var params = req.query;
req.clean.types = req.clean.types || {};
// ensure the input params are a valid object
if( !isObject( params ) ){
params = {};
}
// default case (no layers specified in GET params)
// don't even set the from_layers key in this case
if('string' !== typeof params.source || !params.source.length){
return { error: false };
}
var sources = params.source.split(',');
var invalid_sources = sources.filter(function(source) {
return all_sources.indexOf(source) === -1;
});
if (invalid_sources.length > 0) {
return {
error: true,
msg: '`' + invalid_sources[0] + '` is an invalid source parameter. Valid options: ' + all_sources.join(', ')
};
}
var types = sources.reduce(function(acc, source) {
return acc.concat(sources_map[source]);
}, []);
req.clean.types.from_source = types;
return { error: false };
}
module.exports = sanitize;

1
sanitiser/search.js

@ -4,6 +4,7 @@ var _sanitize = require('../sanitiser/_sanitize'),
input: require('../sanitiser/_input'),
size: require('../sanitiser/_size'),
layers: require('../sanitiser/_layers'),
source: require('../sanitiser/_source'),
details: require('../sanitiser/_details'),
latlonzoom: require('../sanitiser/_geo')
};

55
test/unit/helper/query_parser.js

@ -6,7 +6,8 @@ module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('interface', function(t) {
t.equal(typeof parser, 'function', 'valid function');
t.equal(typeof parser.get_parsed_address, 'function', 'valid function');
t.equal(typeof parser.get_layers, 'function', 'valid function');
t.end();
});
};
@ -17,7 +18,7 @@ module.exports.tests.split_on_comma = function(test, common) {
var testParse = function(query) {
test('naive parsing ' + query, function(t) {
var address = parser(query);
var address = parser.get_parsed_address(query);
var delimIndex = query.indexOf(delim);
var name = query.substring(0, delimIndex);
var admin_parts = query.substring(delimIndex + 1).trim();
@ -41,11 +42,12 @@ module.exports.tests.parse_three_chars_or_less = function(test, common) {
var testParse = function(query) {
test('query length < 3 (' + query + ')', function(t) {
var address = parser(query);
var address = parser.get_parsed_address(query);
var target_layer = get_layers(['admin']);
var layers = parser.get_layers(query);
t.equal(typeof address, 'object', 'valid object');
t.deepEqual(address.target_layer, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
t.end();
});
};
@ -63,17 +65,18 @@ module.exports.tests.parse_one_or_more_tokens = function(test, common) {
var testParse = function(query, parse_address) {
test('query with one or more tokens (' + query + ')', function(t) {
var address = parser(query);
var address = parser.get_parsed_address(query);
var target_layer = get_layers(['admin', 'poi']);
var layers = parser.get_layers(query);
t.equal(typeof address, 'object', 'valid object');
if (parse_address) {
t.deepEqual(address.regions.join(''), query, 'since query contained a number, it went through address parsing');
} else {
t.deepEqual(address.target_layer, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
}
t.end();
});
};
@ -88,33 +91,33 @@ module.exports.tests.parse_one_or_more_tokens = function(test, common) {
};
module.exports.tests.parse_address = function(test, common) {
var addresses_nonum = [{ non_street: 'main particle', city: 'new york'},
{ non_street: 'biggg city block' },
var addresses_nonum = [{ non_street: 'main particle', city: 'new york'},
{ non_street: 'biggg city block' },
{ non_street: 'the empire state building' }
];
var address_with_num = [{ number: 123, street: 'main st', city: 'new york', state: 'ny'},
{ number: 456, street: 'pine ave', city: 'san francisco', state: 'CA'},
var address_with_num = [{ number: 123, street: 'main st', city: 'new york', state: 'ny'},
{ number: 456, street: 'pine ave', city: 'san francisco', state: 'CA'},
{ number: 1980, street: 'house st', city: 'hoboken', state: 'NY'}
];
var address_with_zip = [{ number: 1, street: 'main st', city: 'new york', state: 'ny', zip: 10010},
{ number: 4, street: 'ape ave', city: 'san diego', state: 'CA', zip: 98970},
var address_with_zip = [{ number: 1, street: 'main st', city: 'new york', state: 'ny', zip: 10010},
{ number: 4, street: 'ape ave', city: 'san diego', state: 'CA', zip: 98970},
{ number: 19, street: 'house dr', city: 'houston', state: 'TX', zip: 79089}
];
var testParse = function(query, hasNumber, hasZip) {
var testcase = 'parse query with ' + (hasNumber ? 'a house number ': 'no house number ');
var testcase = 'parse query with ' + (hasNumber ? 'a house number ': 'no house number ');
testcase += 'and ' + (hasZip ? 'a zip ' : 'no zip ');
test(testcase, function(t) {
var query_string = '';
for (var k in query) {
for (var k in query) {
query_string += ' ' + query[k];
}
// remove leading whitespace
query_string = query_string.substring(1);
var address = parser(query_string);
var address = parser.get_parsed_address(query_string);
var non_address_layer = get_layers(['admin', 'poi']);
t.equal(typeof address, 'object', 'valid object for the address ('+query_string+')');
@ -126,21 +129,17 @@ module.exports.tests.parse_address = function(test, common) {
}
if ((hasNumber || hasZip) && query.street) {
t.equal(typeof address.number, 'number', 'valid house number format (' + address.number + ')');
t.equal(typeof address.number, 'number', 'valid house number format (' + address.number + ')');
t.equal(address.number, query.number, 'correct house number (' + query.number + ')');
t.equal(typeof address.street, 'string', 'valid street name format (' + address.street + ')');
t.equal(typeof address.street, 'string', 'valid street name format (' + address.street + ')');
t.equal(address.street, query.street, 'correct street name (' + query.street + ')');
}
if (hasZip) {
t.equal(typeof address.postalcode, 'number', 'valid zip (' + address.postalcode + ')');
t.equal(typeof address.postalcode, 'number', 'valid zip (' + address.postalcode + ')');
t.equal(address.postalcode, query.zip, 'correct postal code (' + query.zip + ')');
}
if (address.text === address.regions.join(' ')) {
t.deepEqual(address.target_layer, query.target_layer, 'admin_parts set correctly to ' + query.target_layer.join(', '));
}
t.end();
});
};
@ -165,4 +164,4 @@ module.exports.all = function (tape, common) {
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};
};

96
test/unit/helper/types.js

@ -0,0 +1,96 @@
var types = require('../../../helper/types');
var valid_types = require( '../../../query/types' );
module.exports.tests = {};
module.exports.tests.no_cleaned_types = function(test, common) {
test('no cleaned types', function(t) {
var actual = types(undefined);
t.equal(actual, undefined, 'all valid types returned for empty input');
t.end();
});
test('no cleaned types', function(t) {
var cleaned_types = {};
var actual = types(cleaned_types);
t.equal(actual, undefined, 'all valid types returned for empty input');
t.end();
});
};
module.exports.tests.address_parser = function(test, common) {
test('address parser specifies only admin layers', function(t) {
var cleaned_types = {
from_address_parser: ['admin0'] // simplified return value from address parser
};
var actual = types(cleaned_types);
var expected = ['admin0']; // simplified expected value for all admin layers
t.deepEqual(actual, expected, 'only layers specified by address parser returned');
t.end();
});
};
module.exports.tests.layers_parameter = function(test, common) {
test('layers parameter specifies only some layers', function(t) {
var cleaned_types = {
from_layers: ['geoname']
};
var actual = types(cleaned_types);
var expected = ['geoname'];
t.deepEqual(actual, expected, 'only types specified by layers parameter returned');
t.end();
});
};
module.exports.tests.layers_parameter_and_address_parser = function(test, common) {
test('layers parameter and address parser present', function(t) {
var cleaned_types = {
from_layers: ['geoname'],
from_address_parser: ['admin0'] // simplified return value from address parse
};
var actual = types(cleaned_types);
var expected = ['geoname'];
t.deepEqual(actual, expected, 'layers parameter overrides address parser completely');
t.end();
});
};
module.exports.tests.source_parameter = function(test, common) {
test('source parameter specified', function(t) {
var cleaned_types = {
from_source: ['openaddresses']
};
var actual = types(cleaned_types);
var expected = ['openaddresses'];
t.deepEqual(actual, expected, 'type parameter set to types specified by source');
t.end();
});
};
module.exports.tests.source_and_layers_parameters = function(test, common) {
test('source and layers parameter both specified', function(t) {
var cleaned_types = {
from_source: ['openaddresses'],
from_layers: ['osmaddress', 'openaddresses']
};
var actual = types(cleaned_types);
var expected = ['openaddresses'];
t.deepEqual(actual, expected, 'type set to intersection of source and layer types');
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('types: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

9
test/unit/query/search.js

@ -274,8 +274,7 @@ module.exports.tests.query = function(test, common) {
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
size: 10,
details: true,
parsed_input: parser(address),
default_layers_set: true
parsed_input: parser.get_parsed_address(address),
});
var expected = {
@ -476,8 +475,7 @@ module.exports.tests.query = function(test, common) {
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
size: 10,
details: true,
parsed_input: parser(partial_address),
default_layers_set: true
parsed_input: parser.get_parsed_address(partial_address),
});
var expected = {
@ -644,8 +642,7 @@ module.exports.tests.query = function(test, common) {
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
size: 10,
details: true,
parsed_input: parser(partial_address),
default_layers_set: true
parsed_input: parser.get_parsed_address(partial_address),
});
var expected = {

10
test/unit/query/indeces.js → test/unit/query/types.js

@ -1,12 +1,12 @@
var indeces = require('../../../query/indeces');
var types = require('../../../query/types');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.true(Array.isArray(indeces), 'valid array');
t.equal(indeces.length, 11, 'valid array');
t.true(Array.isArray(types), 'valid array');
t.equal(types.length, 11, 'valid array');
t.end();
});
};
@ -14,10 +14,10 @@ module.exports.tests.interface = function(test, common) {
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('indeces ' + name, testFunction);
return tape('types ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};
};

4
test/unit/run.js

@ -8,10 +8,11 @@ var tests = [
require('./controller/search'),
require('./service/mget'),
require('./service/search'),
require('./sanitiser/_source'),
require('./sanitiser/search'),
require('./sanitiser/reverse'),
require('./sanitiser/place'),
require('./query/indeces'),
require('./query/types'),
require('./query/sort'),
require('./query/search'),
require('./query/reverse'),
@ -19,6 +20,7 @@ var tests = [
require('./helper/geojsonify'),
require('./helper/outputSchema'),
require('./helper/adminFields'),
require('./helper/types'),
];
tests.map(function(t) {

6
test/unit/sanitiser/_input.js

@ -7,7 +7,7 @@ var input = require('../../../sanitiser/_input'),
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
nonAddressLayers = [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
'locality', 'local_admin' ],
defaultParsed= { target_layer: nonAddressLayers },
defaultParsed= { },
defaultClean = { input: 'test',
layers: allLayers,
size: 10,
@ -17,7 +17,7 @@ var input = require('../../../sanitiser/_input'),
lon:0
},
getTargetLayers = function(query) {
var address = parser(query);
var address = parser.get_parsed_address(query);
return address.target_layer;
};
@ -26,4 +26,4 @@ module.exports = {
defaultParsed: defaultParsed,
defaultClean : defaultClean,
getTargetLayers: getTargetLayers
};
};

114
test/unit/sanitiser/_source.js

@ -0,0 +1,114 @@
var sanitize = require( '../../../sanitiser/_source' );
var success_response = { error: false };
module.exports.tests = {};
module.exports.tests.no_sources = function(test, common) {
test('source is not set', function(t) {
var req = {
query: { }
};
var response = sanitize(req);
t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
t.deepEqual(response, success_response, 'no error returned');
t.end();
});
test('source is empty string', function(t) {
var req = {
query: {
source: ''
}
};
var response = sanitize(req);
t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
t.deepEqual(response, success_response, 'no error returned');
t.end();
});
};
module.exports.tests.valid_sources = function(test, common) {
test('geonames source', function(t) {
var req = {
query: {
source: 'geonames'
}
};
var response = sanitize(req);
t.deepEqual(req.clean.types, { from_source: ['geoname'] }, 'clean.types should contain from_source entry with geonames');
t.deepEqual(response, success_response, 'no error returned');
t.end();
});
test('openstreetmap source', function(t) {
var req = {
query: {
source: 'openstreetmap'
}
};
var expected_types = {
from_source: ['osmaddress', 'osmnode', 'osmway']
};
var response = sanitize(req);
t.deepEqual(req.clean.types, expected_types, 'clean.types should contain from_source entry with multiple entries for openstreetmap');
t.deepEqual(response, success_response, 'no error returned');
t.end();
});
test('multiple sources', function(t) {
var req = {
query: {
source: 'openstreetmap,openaddresses'
}
};
var expected_types = {
from_source: ['osmaddress', 'osmnode', 'osmway', 'openaddresses']
};
var response = sanitize(req);
t.deepEqual(req.clean.types, expected_types,
'clean.types should contain from_source entry with multiple entries for openstreetmap and openadresses');
t.deepEqual(response, success_response, 'no error returned');
t.end();
});
};
module.exports.tests.invalid_sources = function(test, common) {
test('geonames source', function(t) {
var req = {
query: {
source: 'notasource'
}
};
var expected_response = {
error: true,
msg: '`notasource` is an invalid source parameter. Valid options: geonames, openaddresses, quattroshapes, openstreetmap'
};
var response = sanitize(req);
t.deepEqual(response, expected_response, 'error with message returned');
t.deepEqual(req.clean.types, { }, 'clean.types should remain empty');
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANTIZE _source ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

4
test/unit/sanitiser/place.js

@ -2,14 +2,14 @@
var place = require('../../../sanitiser/place'),
_sanitize = place.sanitize,
middleware = place.middleware,
indeces = require('../../../query/indeces'),
types = require('../../../query/types'),
delimiter = ':',
defaultLengthError = function(input) { return 'invalid param \''+ input + '\': text length, must be >0'; },
defaultFormatError = 'invalid: must be of the format type:id for ex: \'geoname:4163334\'',
defaultError = 'invalid param \'id\': text length, must be >0',
defaultMissingTypeError = function(input) {
var type = input.split(delimiter)[0];
return type + ' is invalid. It must be one of these values - [' + indeces.join(', ') + ']'; },
return type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']'; },
defaultClean = { ids: [ { id: '123', type: 'geoname' } ], details: true },
sanitize = function(query, cb) { _sanitize({'query':query}, cb); },
inputs = {

23
test/unit/sanitiser/reverse.js

@ -5,12 +5,11 @@ var suggest = require('../../../sanitiser/reverse'),
delim = ',',
defaultError = 'missing param \'lat\'',
defaultClean = { lat:0,
layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
types: {
},
lon: 0,
size: 10,
details: true,
default_layers_set: true,
categories: []
},
sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
@ -158,7 +157,7 @@ module.exports.tests.sanitize_details = function(test, common) {
module.exports.tests.sanitize_layers = function(test, common) {
test('unspecified', function(t) {
sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
t.end();
});
});
@ -173,21 +172,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
test('poi (alias) layer', function(t) {
var poi_layers = ['geoname','osmnode','osmway'];
sanitize({ layers: 'poi', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, poi_layers, 'poi layers set');
t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
t.end();
});
});
test('admin (alias) layer', function(t) {
var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
sanitize({ layers: 'admin', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, admin_layers, 'admin layers set');
t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
t.end();
});
});
test('address (alias) layer', function(t) {
var address_layers = ['osmaddress','openaddresses'];
sanitize({ layers: 'address', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, address_layers, 'address layers set');
t.deepEqual(clean.types.from_layers, address_layers, 'address layers set');
t.end();
});
});
@ -195,7 +194,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
var poi_layers = ['geoname','osmnode','osmway'];
var reg_layers = ['admin0', 'admin1'];
sanitize({ layers: 'poi,admin0,admin1', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(poi_layers), 'poi + regular layers');
t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
t.end();
});
});
@ -203,7 +202,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
var reg_layers = ['geoname', 'osmway'];
sanitize({ layers: 'admin,geoname,osmway', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
t.end();
});
});
@ -211,21 +210,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
var address_layers = ['osmaddress','openaddresses'];
var reg_layers = ['geoname', 'osmway'];
sanitize({ layers: 'address,geoname,osmway', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(address_layers), 'address + regular layers set');
t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
t.end();
});
});
test('alias layer plus regular layers (no duplicates)', function(t) {
var poi_layers = ['geoname','osmnode','osmway'];
sanitize({ layers: 'poi,geoname,osmnode', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, poi_layers, 'poi layers found (no duplicates)');
t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
t.end();
});
});
test('multiple alias layers (no duplicates)', function(t) {
var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
sanitize({ layers: 'poi,admin', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, alias_layers, 'all layers found (no duplicates)');
t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
t.end();
});
});

117
test/unit/sanitiser/search.js

@ -8,12 +8,11 @@ var search = require('../../../sanitiser/search'),
delim = ',',
defaultError = 'invalid param \'input\': text length, must be >0',
defaultClean = { input: 'test',
layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
types: {
},
size: 10,
details: true,
parsed_input: defaultParsed,
default_layers_set: true
},
sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
@ -32,13 +31,10 @@ module.exports.tests.interface = function(test, common) {
});
};
module.exports.tests.sanitize_input = function(test, common) {
var inputs = {
invalid: [ '', 100, null, undefined, new Date() ],
valid: [ 'a', 'aa', 'aaaaaaaa' ]
};
module.exports.tests.sanitize_invalid_input = function(test, common) {
test('invalid input', function(t) {
inputs.invalid.forEach( function( input ){
var invalid = [ '', 100, null, undefined, new Date() ];
invalid.forEach( function( input ){
sanitize({ input: input }, function( err, clean ){
t.equal(err, 'invalid param \'input\': text length, must be >0', input + ' is an invalid input');
t.equal(clean, undefined, 'clean not set');
@ -46,16 +42,26 @@ module.exports.tests.sanitize_input = function(test, common) {
});
t.end();
});
test('valid input', function(t) {
inputs.valid.forEach( function( input ){
sanitize({ input: input }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.input = input;
expected.parsed_input.target_layer = _input.getTargetLayers(input);
};
t.equal(err, undefined, 'no error');
t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
});
module.exports.tests.sanitise_valid_input = function(test, common) {
test('valid short input', function(t) {
sanitize({ input: 'a' }, function( err, clean ){
t.equal(err, undefined, 'no error');
});
t.end();
});
test('valid not-quite-as-short input', function(t) {
sanitize({ input: 'aa' }, function( err, clean ){
t.equal(err, undefined, 'no error');
});
t.end();
});
test('valid longer input', function(t) {
sanitize({ input: 'aaaaaaaa' }, function( err, clean ){
t.equal(err, undefined, 'no error');
});
t.end();
});
@ -70,11 +76,9 @@ module.exports.tests.sanitize_input_with_delim = function(test, common) {
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.input = input;
expected.parsed_input = parser(input);
expected.parsed_input = parser.get_parsed_address(input);
t.equal(err, undefined, 'no error');
t.equal(clean.parsed_input.name, expected.parsed_input.name, 'clean name set correctly');
t.equal(clean.parsed_input.admin_parts, expected.parsed_input.admin_parts, 'clean admin_parts set correctly');
t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
});
});
@ -101,10 +105,8 @@ module.exports.tests.sanitize_lat = function(test, common) {
sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.lat = parseFloat( lat );
expected.lon = 0;
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set correctly (' + lat + ')');
t.deepEqual(clean.lat, expected.lat, 'clean lat set correctly (' + lat + ')');
});
});
t.end();
@ -120,10 +122,8 @@ module.exports.tests.sanitize_lon = function(test, common) {
sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.lon = parseFloat( lon );
expected.lat = 0;
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set correctly (' + lon + ')');
t.deepEqual(clean.lon, expected.lon, 'clean set correctly (' + lon + ')');
});
});
t.end();
@ -131,34 +131,27 @@ module.exports.tests.sanitize_lon = function(test, common) {
};
module.exports.tests.sanitize_optional_geo = function(test, common) {
test('no lat/lon', function(t) {
test('no lat/lon', function(t) {
sanitize({ input: 'test' }, function( err, clean ){
var expected = defaultClean;
t.equal(err, undefined, 'no error');
t.equal(clean.lat, undefined, 'clean set without lat');
t.equal(clean.lon, undefined, 'clean set without lon');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set without lat/lon');
});
t.end();
});
test('no lat', function(t) {
test('no lat', function(t) {
sanitize({ input: 'test', lon: 0 }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.lon = 0;
var expected_lon = 0;
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set correctly (without any lat)');
t.deepEqual(clean.lon, expected_lon, 'clean set correctly (without any lat)');
});
t.end();
});
test('no lon', function(t) {
test('no lon', function(t) {
sanitize({ input: 'test', lat: 0 }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
expected.lat = 0;
var expected_lat = 0;
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set correctly (without any lon)');
t.deepEqual(clean.lat, expected_lat, 'clean set correctly (without any lon)');
});
t.end();
});
@ -166,8 +159,6 @@ module.exports.tests.sanitize_optional_geo = function(test, common) {
module.exports.tests.sanitize_bbox = function(test, common) {
var bboxes = {
invalid_coordinates: [
],
invalid: [
'91;-181,-91,181', // invalid - semicolon between coordinates
'these,are,not,numbers',
@ -194,22 +185,11 @@ module.exports.tests.sanitize_bbox = function(test, common) {
]
};
test('invalid bbox coordinates', function(t) {
bboxes.invalid_coordinates.forEach( function( bbox ){
sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
t.ok(err.match(/Invalid (lat|lon)/), bbox + ' is invalid');
t.equal(clean, undefined, 'clean not set');
});
});
t.end();
});
test('invalid bbox', function(t) {
bboxes.invalid.forEach( function( bbox ){
sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'falling back on 50km distance from centroid');
t.equal(clean.bbox, undefined, 'falling back on 50km distance from centroid');
});
});
t.end();
@ -217,19 +197,17 @@ module.exports.tests.sanitize_bbox = function(test, common) {
test('valid bbox', function(t) {
bboxes.valid.forEach( function( bbox ){
sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
var expected = JSON.parse(JSON.stringify( defaultClean ));
var bboxArray = bbox.split(',').map(function(i) {
return parseInt(i);
});
expected.bbox = {
var expected_bbox = {
right: Math.max(bboxArray[0], bboxArray[2]),
top: Math.max(bboxArray[1], bboxArray[3]),
left: Math.min(bboxArray[0], bboxArray[2]),
bottom: Math.min(bboxArray[1], bboxArray[3])
};
t.equal(err, undefined, 'no error');
expected.parsed_input = parser('test');
t.deepEqual(clean, expected, 'clean set correctly (' + bbox + ')');
t.deepEqual(clean.bbox, expected_bbox, 'clean set correctly (' + bbox + ')');
});
});
t.end();
@ -320,7 +298,7 @@ module.exports.tests.sanitize_details = function(test, common) {
module.exports.tests.sanitize_layers = function(test, common) {
test('unspecified', function(t) {
sanitize({ layers: undefined, input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
t.end();
});
});
@ -335,21 +313,22 @@ module.exports.tests.sanitize_layers = function(test, common) {
test('poi (alias) layer', function(t) {
var poi_layers = ['geoname','osmnode','osmway'];
sanitize({ layers: 'poi', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, poi_layers, 'poi layers set');
t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
t.end();
});
});
test('admin (alias) layer', function(t) {
var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
sanitize({ layers: 'admin', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, admin_layers, 'admin layers set');
t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
t.end();
});
});
test('address (alias) layer', function(t) {
var address_layers = ['osmaddress','openaddresses'];
sanitize({ layers: 'address', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, address_layers, 'address layers set');
t.deepEqual(clean.types.from_layers, address_layers, 'types from layers set');
t.deepEqual(clean.types.from_address_parser, _input.allLayers, 'address parser uses default layers');
t.end();
});
});
@ -357,7 +336,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
var poi_layers = ['geoname','osmnode','osmway'];
var reg_layers = ['admin0', 'admin1'];
sanitize({ layers: 'poi,admin0,admin1', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(poi_layers), 'poi + regular layers');
t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
t.end();
});
});
@ -365,7 +344,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
var reg_layers = ['geoname', 'osmway'];
sanitize({ layers: 'admin,geoname,osmway', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
t.end();
});
});
@ -373,21 +352,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
var address_layers = ['osmaddress','openaddresses'];
var reg_layers = ['geoname', 'osmway'];
sanitize({ layers: 'address,geoname,osmway', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, reg_layers.concat(address_layers), 'address + regular layers set');
t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
t.end();
});
});
test('alias layer plus regular layers (no duplicates)', function(t) {
var poi_layers = ['geoname','osmnode','osmway'];
sanitize({ layers: 'poi,geoname,osmnode', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, poi_layers, 'poi layers found (no duplicates)');
t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
t.end();
});
});
test('multiple alias layers (no duplicates)', function(t) {
var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
sanitize({ layers: 'poi,admin', input: 'test' }, function( err, clean ){
t.deepEqual(clean.layers, alias_layers, 'all layers found (no duplicates)');
t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
t.end();
});
});
@ -420,8 +399,6 @@ module.exports.tests.middleware_success = function(test, common) {
var req = { query: { input: 'test' }};
var next = function( message ){
t.equal(message, undefined, 'no error message set');
req.clean.parsed_input = parser('test');
t.deepEqual(req.clean, defaultClean);
t.end();
};
middleware( req, undefined, next );

3
test/unit/sanitiser/suggest.js

@ -15,7 +15,6 @@ var suggest = require('../../../sanitiser/suggest'),
lat:0,
lon:0,
parsed_input: defaultParsed,
default_layers_set: true
},
sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
@ -272,7 +271,7 @@ module.exports.tests.sanitize_details = function(test, common) {
module.exports.tests.sanitize_layers = function(test, common) {
test('unspecified', function(t) {
sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){
t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
t.end();
});
});

Loading…
Cancel
Save