mirror of https://github.com/pelias/api.git
Browse Source
This is functionally the same code as before, except that it's now possible to do better logging for invalid source/layer combinations.wof_focus_weighting
Julian Simioni
9 years ago
26 changed files with 240 additions and 481 deletions
@ -1,87 +1,85 @@
|
||||
var extend = require('extend'), |
||||
_ = require('lodash'); |
||||
|
||||
var TYPE_TO_SOURCE = { |
||||
'geoname': 'gn', |
||||
'osmnode': 'osm', |
||||
'osmway': 'osm', |
||||
'admin0': 'qs', |
||||
'admin1': 'qs', |
||||
'admin2': 'qs', |
||||
'neighborhood': 'qs', |
||||
'locality': 'qs', |
||||
'local_admin': 'qs', |
||||
'osmaddress': 'osm', |
||||
'openaddresses': 'oa' |
||||
}; |
||||
function addStandardTargetsToAliases(standard, aliases) { |
||||
var combined = _.extend({}, aliases); |
||||
standard.forEach(function(target) { |
||||
if (combined[target] === undefined) { |
||||
combined[target] = [target]; |
||||
} |
||||
}); |
||||
|
||||
return combined; |
||||
} |
||||
|
||||
/* |
||||
* This doesn't include alias layers such as coarse |
||||
* Sources |
||||
*/ |
||||
var TYPE_TO_LAYER = { |
||||
'geoname': 'venue', |
||||
'osmnode': 'venue', |
||||
'osmway': 'venue', |
||||
'admin0': 'country', |
||||
'admin1': 'region', |
||||
'admin2': 'county', |
||||
'neighborhood': 'neighbourhood', |
||||
'locality': 'locality', |
||||
'local_admin': 'localadmin', |
||||
'osmaddress': 'address', |
||||
'openaddresses': 'address' |
||||
}; |
||||
|
||||
var SOURCE_TO_TYPE = { |
||||
'gn' : ['geoname'], |
||||
'geonames' : ['geoname'], |
||||
'oa' : ['openaddresses'], |
||||
'openaddresses' : ['openaddresses'], |
||||
'qs' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
'osm' : ['osmaddress', 'osmnode', 'osmway'], |
||||
'openstreetmap' : ['osmaddress', 'osmnode', 'osmway'] |
||||
// a list of all sources
|
||||
var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'quattroshapes', 'whosonfirst']; |
||||
|
||||
/* |
||||
* A list of alternate names for sources, mostly used to save typing |
||||
*/ |
||||
var SOURCE_ALIASES = { |
||||
'osm': ['openstreetmap'], |
||||
'oa': ['openaddresses'], |
||||
'gn': ['geonames'], |
||||
'qs': ['quattroshapes'], |
||||
'wof': ['whosonfirst'] |
||||
}; |
||||
|
||||
/** |
||||
* This does not included alias layers, those are built separately |
||||
/* |
||||
* Create an object that contains all sources or aliases. The key is the source or alias, |
||||
* the value is either that source, or the canonical name for that alias if it's an alias. |
||||
*/ |
||||
var LAYER_TO_TYPE = { |
||||
'venue': ['geoname','osmnode','osmway'], |
||||
'address': ['osmaddress','openaddresses'], |
||||
'country': ['admin0'], |
||||
'region': ['admin1'], |
||||
'county': ['admin2'], |
||||
'locality': ['locality'], |
||||
'localadmin': ['local_admin'], |
||||
'neighbourhood': ['neighborhood'] |
||||
var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES); |
||||
|
||||
/* |
||||
* Layers |
||||
*/ |
||||
|
||||
/* |
||||
* A list of all layers in each source. This is used for convenience elswhere |
||||
* and to determine when a combination of source and layer parameters is |
||||
* not going to match any records and will return no results. |
||||
*/ |
||||
var LAYERS_BY_SOURCE = { |
||||
openstreetmap: [ 'address', 'venue' ], |
||||
openaddresses: [ 'address' ], |
||||
geonames: [ 'country', 'region', 'county', 'locality', 'venue' ], |
||||
quattroshapes: ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'], |
||||
whosonfirst: [ 'continent', 'macrocountry', 'country', 'dependency', 'region', |
||||
'locality', 'localadmin', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed'] |
||||
}; |
||||
|
||||
/* |
||||
* A list of layer aliases that can be used to support specific use cases |
||||
* (like coarse geocoding) * or work around the fact that different sources |
||||
* may have layers that mean the same thing but have a different name |
||||
*/ |
||||
var LAYER_ALIASES = { |
||||
'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin'] |
||||
'coarse': LAYERS_BY_SOURCE.whosonfirst.concat(LAYERS_BY_SOURCE.quattroshapes), |
||||
'country': ['country', 'admin0'], // Include both QS and WOF layers for various types of places
|
||||
'region': ['region', 'admin1'] // Alias layers that include themselves look weird, but do work
|
||||
}; |
||||
|
||||
var LAYER_WITH_ALIASES_TO_TYPE = extend({}, LAYER_ALIASES, LAYER_TO_TYPE); |
||||
// create a list of all layers by combining each entry from LAYERS_BY_SOURCE
|
||||
var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) { |
||||
return acc.concat(LAYERS_BY_SOURCE[key]); |
||||
}, [])); |
||||
|
||||
/* |
||||
* derive the list of types, sources, and layers from above mappings |
||||
* Create the an object that has a key for each possible layer or alias, |
||||
* and returns either that layer, or all the layers in the alias |
||||
*/ |
||||
var TYPES = Object.keys(TYPE_TO_SOURCE); |
||||
var SOURCES = Object.keys(SOURCE_TO_TYPE); |
||||
var LAYERS = Object.keys(LAYER_TO_TYPE); |
||||
|
||||
var sourceAndLayerToType = function sourceAndLayerToType(source, layer) { |
||||
return _.intersection(SOURCE_TO_TYPE[source], LAYER_WITH_ALIASES_TO_TYPE[layer]); |
||||
}; |
||||
var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES); |
||||
|
||||
module.exports = { |
||||
types: TYPES, |
||||
sources: SOURCES, |
||||
layers: LAYERS, |
||||
type_to_source: TYPE_TO_SOURCE, |
||||
type_to_layer: TYPE_TO_LAYER, |
||||
source_to_type: SOURCE_TO_TYPE, |
||||
layer_to_type: LAYER_TO_TYPE, |
||||
layer_with_aliases_to_type: LAYER_WITH_ALIASES_TO_TYPE, |
||||
source_and_layer_to_type: sourceAndLayerToType |
||||
source_mapping: SOURCE_MAPPING, |
||||
layer_mapping: LAYER_MAPPING, |
||||
layers_by_source: LAYERS_BY_SOURCE |
||||
}; |
||||
|
||||
@ -1,43 +0,0 @@
|
||||
var type_mapping = require( '../helper/type_mapping' ); |
||||
var _ = require('lodash'); |
||||
|
||||
/** |
||||
* Different parts of the code express "preferences" for which Elasticsearch types are going to be searched |
||||
* This method decides how to combine all the preferences. |
||||
* |
||||
* @param {Array} clean_types |
||||
* @returns {Array} |
||||
*/ |
||||
module.exports = function calculate_types(clean_types) { |
||||
//Check that at least one preference of types is defined
|
||||
if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) { |
||||
throw new Error('clean_types should not be null or undefined'); |
||||
} |
||||
|
||||
/* the layers and source parameters are cumulative: |
||||
* perform a set intersection of their specified types |
||||
*/ |
||||
if (clean_types.from_layers || clean_types.from_sources) { |
||||
var types = type_mapping.types; |
||||
|
||||
if (clean_types.from_layers) { |
||||
types = _.intersection(types, clean_types.from_layers); |
||||
} |
||||
|
||||
if (clean_types.from_sources) { |
||||
types = _.intersection(types, clean_types.from_sources); |
||||
} |
||||
|
||||
return types; |
||||
} |
||||
|
||||
/* |
||||
* Type restrictions requested by the address parser should only be used |
||||
* if both the source and layers parameters are empty, so do this last |
||||
*/ |
||||
if (clean_types.from_text_parser) { |
||||
return clean_types.from_text_parser; |
||||
} |
||||
|
||||
throw new Error('no types specified'); |
||||
}; |
||||
@ -0,0 +1,32 @@
|
||||
var _ = require( 'lodash' ); |
||||
var type_mapping = require( '../helper/type_mapping' ); |
||||
function sanitize( raw, clean ){ |
||||
var messages = { errors: [], warnings: [] }; |
||||
|
||||
var possible_errors = []; |
||||
var at_least_one_valid_combination = false; |
||||
|
||||
if (clean.layers && clean.sources) { |
||||
clean.sources.forEach(function(source) { |
||||
var layers_for_source = type_mapping.layers_by_source[source]; |
||||
clean.layers.forEach(function(layer) { |
||||
if (_.includes(layers_for_source, layer)) { |
||||
at_least_one_valid_combination = true; |
||||
} else { |
||||
var message = 'You have specified both the `sources` and `layers`' + |
||||
'parameters in a combination that will return no results: the ' + |
||||
source + ' source has nothing in the ' + layer + ' layer'; |
||||
possible_errors.push(message); |
||||
} |
||||
}); |
||||
}); |
||||
|
||||
if (!at_least_one_valid_combination) { |
||||
messages.errors = possible_errors; |
||||
} |
||||
} |
||||
|
||||
return messages; |
||||
} |
||||
|
||||
module.exports = sanitize; |
||||
@ -1,92 +0,0 @@
|
||||
var types = require('../../../helper/types'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.no_cleaned_types = function(test, common) { |
||||
test('no cleaned types', function(t) { |
||||
function testIt() { |
||||
types({}); |
||||
} |
||||
|
||||
t.throws(testIt, /clean_types should not be null or undefined/, 'no input should result in exception'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.address_parser = function(test, common) { |
||||
test('address parser specifies only admin layers', function(t) { |
||||
var cleaned_types = { |
||||
from_text_parser: ['admin0'] // simplified return value from address parser
|
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['admin0']; // simplified expected value for all admin layers
|
||||
t.deepEqual(actual, expected, 'only layers specified by address parser returned'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.layers_parameter = function(test, common) { |
||||
test('layers parameter specifies only some layers', function(t) { |
||||
var cleaned_types = { |
||||
from_layers: ['geoname'] |
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['geoname']; |
||||
t.deepEqual(actual, expected, 'only types specified by layers parameter returned'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.layers_parameter_and_address_parser = function(test, common) { |
||||
test('layers parameter and address parser present', function(t) { |
||||
var cleaned_types = { |
||||
from_layers: ['geoname'], |
||||
from_text_parser: ['admin0'] // simplified return value from address parse
|
||||
}; |
||||
var actual = types(cleaned_types); |
||||
var expected = ['geoname']; |
||||
t.deepEqual(actual, expected, 'layers parameter overrides address parser completely'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.source_parameter = function(test, common) { |
||||
test('source parameter specified', function(t) { |
||||
var cleaned_types = { |
||||
from_sources: ['openaddresses'] |
||||
}; |
||||
|
||||
var actual = types(cleaned_types); |
||||
|
||||
var expected = ['openaddresses']; |
||||
t.deepEqual(actual, expected, 'type parameter set to types specified by source'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.source_and_layers_parameters = function(test, common) { |
||||
test('source and layers parameter both specified', function(t) { |
||||
var cleaned_types = { |
||||
from_sources: ['openaddresses'], |
||||
from_layers: ['osmaddress', 'openaddresses'] |
||||
}; |
||||
|
||||
var actual = types(cleaned_types); |
||||
|
||||
var expected = ['openaddresses']; |
||||
t.deepEqual(actual, expected, 'type set to intersection of source and layer types'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
|
||||
function test(name, testFunction) { |
||||
return tape('types: ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue