mirror of https://github.com/pelias/api.git
Julian Simioni
9 years ago
62 changed files with 463 additions and 717 deletions
@ -1,87 +1,85 @@
|
||||
var extend = require('extend'), |
||||
_ = require('lodash'); |
||||
|
||||
var TYPE_TO_SOURCE = { |
||||
'geoname': 'gn', |
||||
'osmnode': 'osm', |
||||
'osmway': 'osm', |
||||
'admin0': 'qs', |
||||
'admin1': 'qs', |
||||
'admin2': 'qs', |
||||
'neighborhood': 'qs', |
||||
'locality': 'qs', |
||||
'local_admin': 'qs', |
||||
'osmaddress': 'osm', |
||||
'openaddresses': 'oa' |
||||
}; |
||||
function addStandardTargetsToAliases(standard, aliases) { |
||||
var combined = _.extend({}, aliases); |
||||
standard.forEach(function(target) { |
||||
if (combined[target] === undefined) { |
||||
combined[target] = [target]; |
||||
} |
||||
}); |
||||
|
||||
return combined; |
||||
} |
||||
|
||||
/* |
||||
* This doesn't include alias layers such as coarse |
||||
* Sources |
||||
*/ |
||||
var TYPE_TO_LAYER = { |
||||
'geoname': 'venue', |
||||
'osmnode': 'venue', |
||||
'osmway': 'venue', |
||||
'admin0': 'country', |
||||
'admin1': 'region', |
||||
'admin2': 'county', |
||||
'neighborhood': 'neighbourhood', |
||||
'locality': 'locality', |
||||
'local_admin': 'localadmin', |
||||
'osmaddress': 'address', |
||||
'openaddresses': 'address' |
||||
}; |
||||
|
||||
var SOURCE_TO_TYPE = { |
||||
'gn' : ['geoname'], |
||||
'geonames' : ['geoname'], |
||||
'oa' : ['openaddresses'], |
||||
'openaddresses' : ['openaddresses'], |
||||
'qs' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
'osm' : ['osmaddress', 'osmnode', 'osmway'], |
||||
'openstreetmap' : ['osmaddress', 'osmnode', 'osmway'] |
||||
// a list of all sources
|
||||
var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'quattroshapes', 'whosonfirst']; |
||||
|
||||
/* |
||||
* A list of alternate names for sources, mostly used to save typing |
||||
*/ |
||||
var SOURCE_ALIASES = { |
||||
'osm': ['openstreetmap'], |
||||
'oa': ['openaddresses'], |
||||
'gn': ['geonames'], |
||||
'qs': ['quattroshapes'], |
||||
'wof': ['whosonfirst'] |
||||
}; |
||||
|
||||
/** |
||||
* This does not included alias layers, those are built separately |
||||
/* |
||||
* Create an object that contains all sources or aliases. The key is the source or alias, |
||||
* the value is either that source, or the canonical name for that alias if it's an alias. |
||||
*/ |
||||
var LAYER_TO_TYPE = { |
||||
'venue': ['geoname','osmnode','osmway'], |
||||
'address': ['osmaddress','openaddresses'], |
||||
'country': ['admin0'], |
||||
'region': ['admin1'], |
||||
'county': ['admin2'], |
||||
'locality': ['locality'], |
||||
'localadmin': ['local_admin'], |
||||
'neighbourhood': ['neighborhood'] |
||||
var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES); |
||||
|
||||
/* |
||||
* Layers |
||||
*/ |
||||
|
||||
/* |
||||
* A list of all layers in each source. This is used for convenience elswhere |
||||
* and to determine when a combination of source and layer parameters is |
||||
* not going to match any records and will return no results. |
||||
*/ |
||||
var LAYERS_BY_SOURCE = { |
||||
openstreetmap: [ 'address', 'venue' ], |
||||
openaddresses: [ 'address' ], |
||||
geonames: [ 'country', 'region', 'county', 'locality', 'venue' ], |
||||
quattroshapes: ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region', |
||||
'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed'] |
||||
}; |
||||
|
||||
/* |
||||
* A list of layer aliases that can be used to support specific use cases |
||||
* (like coarse geocoding) * or work around the fact that different sources |
||||
* may have layers that mean the same thing but have a different name |
||||
*/ |
||||
var LAYER_ALIASES = { |
||||
'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin'] |
||||
'coarse': LAYERS_BY_SOURCE.whosonfirst.concat(LAYERS_BY_SOURCE.quattroshapes), |
||||
'country': ['country', 'admin0'], // Include both QS and WOF layers for various types of places
|
||||
'region': ['region', 'admin1'] // Alias layers that include themselves look weird, but do work
|
||||
}; |
||||
|
||||
var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE); |
||||
// create a list of all layers by combining each entry from LAYERS_BY_SOURCE
|
||||
var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) { |
||||
return acc.concat(LAYERS_BY_SOURCE[key]); |
||||
}, [])); |
||||
|
||||
/* |
||||
* derive the list of types, sources, and layers from above mappings |
||||
* Create the an object that has a key for each possible layer or alias, |
||||
* and returns either that layer, or all the layers in the alias |
||||
*/ |
||||
var TYPES = Object.keys(TYPE_TO_SOURCE); |
||||
var SOURCES = Object.keys(SOURCE_TO_TYPE); |
||||
var LAYERS = Object.keys(LAYER_TO_TYPE); |
||||
|
||||
var sourceAndLayerToType = function sourceAndLayerToType(source, layer) { |
||||
return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]); |
||||
}; |
||||
var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES); |
||||
|
||||
module.exports = { |
||||
types: TYPES, |
||||
sources: SOURCES, |
||||
layers: LAYERS, |
||||
type_to_source: TYPE_TO_SOURCE, |
||||
type_to_layer: TYPE_TO_LAYER, |
||||
source_to_type: SOURCE_TO_TYPE, |
||||
layer_to_type: LAYER_TO_TYPE, |
||||
layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE, |
||||
source_and_layer_to_type: sourceAndLayerToType |
||||
source_mapping: SOURCE_MAPPING, |
||||
layer_mapping: LAYER_MAPPING, |
||||
layers_by_source: LAYERS_BY_SOURCE |
||||
}; |
||||
|
||||
@ -1,43 +0,0 @@
|
||||
var type_mapping = require( '../helper/type_mapping' ); |
||||
var _ = require('lodash'); |
||||
|
||||
/** |
||||
* Different parts of the code express "preferences" for which Elasticsearch types are going to be searched |
||||
* This method decides how to combine all the preferences. |
||||
* |
||||
* @param {Array} clean_types |
||||
* @returns {Array} |
||||
*/ |
||||
module.exports = function calculate_types(clean_types) { |
||||
//Check that at least one preference of types is defined
|
||||
if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) { |
||||
throw new Error('clean_types should not be null or undefined'); |
||||
} |
||||
|
||||
/* the layers and source parameters are cumulative: |
||||
* perform a set intersection of their specified types |
||||
*/ |
||||
if (clean_types.from_layers || clean_types.from_sources) { |
||||
var types = type_mapping.types; |
||||
|
||||
if (clean_types.from_layers) { |
||||
types = _.intersection(types, clean_types.from_layers); |
||||
} |
||||
|
||||
if (clean_types.from_sources) { |
||||
types = _.intersection(types, clean_types.from_sources); |
||||
} |
||||
|
||||
return types; |
||||
} |
||||
|
||||
/* |
||||
* Type restrictions requested by the address parser should only be used |
||||
* if both the source and layers parameters are empty, so do this last |
||||
*/ |
||||
if (clean_types.from_text_parser) { |
||||
return clean_types.from_text_parser; |
||||
} |
||||
|
||||
throw new Error('no types specified'); |
||||
}; |
||||
@ -1,46 +0,0 @@
|
||||
var types_helper = require( '../helper/types' ); |
||||
|
||||
/** |
||||
* Validate the types specified to be searched. |
||||
* |
||||
* Elasticsearch interprets an empty array of types as "search anything" rather |
||||
* than "search nothing", so in the case of an empty array, return an error |
||||
* message instead of searching at all. |
||||
*/ |
||||
function middleware(req, res, next) { |
||||
req.clean = req.clean || {}; |
||||
|
||||
if (req.clean.hasOwnProperty('types')) { |
||||
|
||||
try { |
||||
var types = types_helper(req.clean.types); |
||||
|
||||
if ((types instanceof Array) && types.length === 0) { |
||||
var err = 'You have specified both the `sources` and `layers` ' + |
||||
'parameters in a combination that will return no results.'; |
||||
req.errors.push( err ); |
||||
} |
||||
|
||||
else { |
||||
req.clean.type = types; |
||||
} |
||||
|
||||
} |
||||
|
||||
// @todo: refactor this flow, it is confusing as `types_helper()` can throw
|
||||
// with an error "clean_types should not be null or undefined" which is
|
||||
// not returned to the user yet the return value CAN trigger a user error.
|
||||
// I would have liked to throw for BOTH cases and then handle the users errors
|
||||
// inside the 'catch' but this is not possible.
|
||||
// also: why are we deleting things from $clean?
|
||||
catch (err) { |
||||
// this means there were no types specified
|
||||
delete req.clean.types; |
||||
} |
||||
|
||||
} |
||||
|
||||
next(); |
||||
} |
||||
|
||||
module.exports = middleware; |
||||
@ -1,46 +0,0 @@
|
||||
|
||||
var _ = require('lodash'), |
||||
check = require('check-types'), |
||||
sources_map = require( '../query/sources' ); |
||||
|
||||
var ALL_SOURCES = Object.keys(sources_map), |
||||
ALL_SOURCES_JOINED = ALL_SOURCES.join(','); |
||||
|
||||
function sanitize( raw, clean ) { |
||||
|
||||
// error & warning messages
|
||||
var messages = { errors: [], warnings: [] }; |
||||
|
||||
// init clean.types (if not already init)
|
||||
clean.types = clean.types || {}; |
||||
|
||||
// default case (no layers specified in GET params)
|
||||
// don't even set the from_layers key in this case
|
||||
if( check.nonEmptyString( raw.source ) ){ |
||||
|
||||
var sources = raw.source.split(','); |
||||
|
||||
var invalid_sources = sources.filter(function(source) { |
||||
return !_.includes( ALL_SOURCES, source ); |
||||
}); |
||||
|
||||
if( invalid_sources.length > 0 ){ |
||||
invalid_sources.forEach( function( invalid ){ |
||||
messages.errors.push('\'' + invalid + '\' is an invalid source parameter. Valid options: ' + ALL_SOURCES_JOINED); |
||||
}); |
||||
} |
||||
|
||||
else { |
||||
var types = sources.reduce(function(acc, source) { |
||||
return acc.concat(sources_map[source]); |
||||
}, []); |
||||
|
||||
clean.types.from_source = types; |
||||
} |
||||
|
||||
} |
||||
|
||||
return messages; |
||||
} |
||||
|
||||
module.exports = sanitize; |
||||
@ -0,0 +1,37 @@
|
||||
var _ = require( 'lodash' ); |
||||
var type_mapping = require( '../helper/type_mapping' ); |
||||
|
||||
/* |
||||
* This sanitiser depends on clean.layers and clean.sources |
||||
* so it has to be run after those sanitisers have been run |
||||
*/ |
||||
function sanitize( raw, clean ){ |
||||
var messages = { errors: [], warnings: [] }; |
||||
|
||||
var possible_errors = []; |
||||
var at_least_one_valid_combination = false; |
||||
|
||||
if (clean.layers && clean.sources) { |
||||
clean.sources.forEach(function(source) { |
||||
var layers_for_source = type_mapping.layers_by_source[source]; |
||||
clean.layers.forEach(function(layer) { |
||||
if (_.includes(layers_for_source, layer)) { |
||||
at_least_one_valid_combination = true; |
||||
} else { |
||||
var message = 'You have specified both the `sources` and `layers` ' + |
||||
'parameters in a combination that will return no results: the ' + |
||||
source + ' source has nothing in the ' + layer + ' layer'; |
||||
possible_errors.push(message); |
||||
} |
||||
}); |
||||
}); |
||||
|
||||
if (!at_least_one_valid_combination) { |
||||
messages.errors = possible_errors; |
||||
} |
||||
} |
||||
|
||||
return messages; |
||||
} |
||||
|
||||
module.exports = sanitize; |
||||
@ -1,92 +0,0 @@
|
||||
var types = require('../../../helper/types'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.no_cleaned_types = function(test, common) { |
||||
test('no cleaned types', function(t) { |
||||
function testIt() { |
||||
types({}); |
||||
} |
||||
|
||||
t.throws(testIt, /clean_types should not be null or undefined/, 'no input should result in exception'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.address_parser = function(test, common) { |
||||
test('address parser specifies only admin layers', function(t) { |
||||
var cleaned_types = { |
||||
from_text_parser: ['admin0'] // simplified return value from address parser
|
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['admin0']; // simplified expected value for all admin layers
|
||||
t.deepEqual(actual, expected, 'only layers specified by address parser returned'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.layers_parameter = function(test, common) { |
||||
test('layers parameter specifies only some layers', function(t) { |
||||
var cleaned_types = { |
||||
from_layers: ['geoname'] |
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['geoname']; |
||||
t.deepEqual(actual, expected, 'only types specified by layers parameter returned'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.layers_parameter_and_address_parser = function(test, common) { |
||||
test('layers parameter and address parser present', function(t) { |
||||
var cleaned_types = { |
||||
from_layers: ['geoname'], |
||||
from_text_parser: ['admin0'] // simplified return value from address parse
|
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['geoname']; |
||||
t.deepEqual(actual, expected, 'layers parameter overrides address parser completely'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.source_parameter = function(test, common) { |
||||
test('source parameter specified', function(t) { |
||||
var cleaned_types = { |
||||
from_sources: ['openaddresses'] |
||||
}; |
||||
|
||||
var actual = types(cleaned_types); |
||||
|
||||
var expected = ['openaddresses']; |
||||
t.deepEqual(actual, expected, 'type parameter set to types specified by source'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.source_and_layers_parameters = function(test, common) { |
||||
test('source and layers parameter both specified', function(t) { |
||||
var cleaned_types = { |
||||
from_sources: ['openaddresses'], |
||||
from_layers: ['osmaddress', 'openaddresses'] |
||||
}; |
||||
|
||||
var actual = types(cleaned_types); |
||||
|
||||
var expected = ['openaddresses']; |
||||
t.deepEqual(actual, expected, 'type set to intersection of source and layer types'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
|
||||
function test(name, testFunction) { |
||||
return tape('types: ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
@ -0,0 +1,115 @@
|
||||
var sanitize = require('../../../sanitiser/_sources_and_layers'); |
||||
|
||||
var type_mapping = require('../../../helper/type_mapping'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.inactive = function(test, common) { |
||||
test('no source or layer specified', function(t) { |
||||
var raw = {}; |
||||
var clean = {}; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('only layers specified', function(t) { |
||||
var raw = {}; |
||||
var clean = { layers: ['venue'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('only sources specified', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['openstreetmap'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.no_errors = function(test, common) { |
||||
test('valid combination', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['openstreetmap'], layers: ['venue'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('valid combination because of multiple sources', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['openstreetmap', 'openaddresses'], layers: ['venue'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('valid combination because of multiple layers', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['openaddresses'], layers: ['address', 'country'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 0, 'should return no errors'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.invalid_combination = function(test, common) { |
||||
test('address layer with wof', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['whosonfirst'], layers: ['address'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 1, 'should return an error'); |
||||
t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' + |
||||
'parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('admin layers with osm', function(t) { |
||||
var raw = {}; |
||||
var clean = { sources: ['openstreetmap'], layers: ['country', 'locality'] }; |
||||
|
||||
var messages = sanitize(raw, clean); |
||||
|
||||
t.equal(messages.errors.length, 2, 'should return an error'); |
||||
t.equal(messages.errors[0], 'You have specified both the `sources` and `layers` ' + |
||||
'parameters in a combination that will return no results: the openstreetmap source has nothing in the country layer'); |
||||
t.equal(messages.errors[1], 'You have specified both the `sources` and `layers` ' + |
||||
'parameters in a combination that will return no results: the openstreetmap source has nothing in the locality layer'); |
||||
t.equal(messages.warnings.length, 0, 'should return no warnings'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('SANTIZE _sources_and_layers ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue