mirror of https://github.com/pelias/api.git
Julian Simioni
7 years ago
committed by
GitHub
10 changed files with 536 additions and 107 deletions
@ -0,0 +1,146 @@
|
||||
const _ = require('lodash'); |
||||
const elasticsearch = require('elasticsearch'); |
||||
|
||||
var TypeMapping = function(){ |
||||
|
||||
// A list of all sources
|
||||
this.sources = []; |
||||
|
||||
// A list of alternate names for sources, mostly used to save typing
|
||||
this.source_aliases = {}; |
||||
|
||||
// A list of all layers
|
||||
this.layers = []; |
||||
|
||||
/* |
||||
* A list of all layers in each source. This is used for convenience elswhere |
||||
* and to determine when a combination of source and layer parameters is |
||||
* not going to match any records and will return no results. |
||||
*/ |
||||
this.layers_by_source = {}; |
||||
|
||||
/* |
||||
* A list of layer aliases that can be used to support specific use cases |
||||
* (like coarse geocoding) * or work around the fact that different sources |
||||
* may have layers that mean the same thing but have a different name |
||||
*/ |
||||
this.layer_aliases = {}; |
||||
|
||||
/* |
||||
* An object that contains all sources or aliases. The key is the source or alias, |
||||
* the value is either that source, or the canonical name for that alias if it's an alias. |
||||
*/ |
||||
this.source_mapping = {}; |
||||
|
||||
/* |
||||
* An object that has a key for each possible layer or alias, |
||||
* and returns either that layer, or gall the layers in the alias |
||||
*/ |
||||
this.layer_mapping = {}; |
||||
}; |
||||
|
||||
TypeMapping.addStandardTargetsToAliases = function(standard, aliases) { |
||||
var combined = _.extend({}, aliases); |
||||
standard.forEach(function(target) { |
||||
if (combined[target] === undefined) { |
||||
combined[target] = [target]; |
||||
} |
||||
}); |
||||
|
||||
return combined; |
||||
}; |
||||
|
||||
// source alias setter
|
||||
TypeMapping.prototype.setSourceAliases = function( aliases ){ |
||||
this.source_aliases = aliases; |
||||
}; |
||||
|
||||
// layers-by-source alias setter
|
||||
TypeMapping.prototype.setLayersBySource = function( lbs ){ |
||||
this.layers_by_source = lbs; |
||||
}; |
||||
|
||||
// layer alias setter
|
||||
TypeMapping.prototype.setLayerAliases = function( aliases ){ |
||||
this.layer_aliases = aliases; |
||||
}; |
||||
|
||||
// generate mappings after setters have been run
|
||||
TypeMapping.prototype.generateMappings = function(){ |
||||
this.sources = Object.keys( this.layers_by_source ); |
||||
this.source_mapping = TypeMapping.addStandardTargetsToAliases(this.sources, this.source_aliases); |
||||
this.layers = _.uniq(Object.keys(this.layers_by_source).reduce(function(acc, key) { |
||||
return acc.concat(this.layers_by_source[key]); |
||||
}.bind(this), [])); |
||||
this.layer_mapping = TypeMapping.addStandardTargetsToAliases(this.layers, this.layer_aliases); |
||||
}; |
||||
|
||||
// load values from targets block
|
||||
TypeMapping.prototype.loadTargets = function( targetsBlock ){ |
||||
|
||||
if( !_.isObject(targetsBlock) ){ return; } |
||||
|
||||
// set values from targets block
|
||||
this.setSourceAliases( targetsBlock.source_aliases || {} ); |
||||
this.setLayersBySource( targetsBlock.layers_by_source || {} ); |
||||
this.setLayerAliases( targetsBlock.layer_aliases || {} ); |
||||
|
||||
// generate the mappings
|
||||
this.generateMappings(); |
||||
}; |
||||
|
||||
// load values from either pelias config file or from elasticsearch
|
||||
TypeMapping.prototype.load = function( done ){ |
||||
|
||||
// load pelias config
|
||||
const peliasConfigTargets = _.get( |
||||
require('pelias-config').generate(require('../schema')), |
||||
'api.targets', {} |
||||
); |
||||
|
||||
// load targets from config file
|
||||
this.loadTargets( peliasConfigTargets ); |
||||
|
||||
// do not load values from elasticsearch
|
||||
if( true !== peliasConfigTargets.auto_discover ){ |
||||
if( 'function' === typeof done ){ done(); } |
||||
return; |
||||
} |
||||
|
||||
if( 'function' === typeof done ){ done(); } |
||||
return; |
||||
|
||||
// load values from elasticsearch
|
||||
|
||||
// create connection to elasticsearch
|
||||
// const esclient = elasticsearch.Client(peliasConfig.esclient);
|
||||
|
||||
// const query = {
|
||||
// requestCache: true,
|
||||
// preference: '_replica_first',
|
||||
// timeout: '10s',
|
||||
// body: {
|
||||
// aggs: {
|
||||
// sources: {
|
||||
// terms: {
|
||||
// field: 'source',
|
||||
// size: 100
|
||||
// }
|
||||
// },
|
||||
// layers: {
|
||||
// terms: {
|
||||
// field: 'layer',
|
||||
// size: 100
|
||||
// }
|
||||
// }
|
||||
// },
|
||||
// size: 0
|
||||
// }
|
||||
// };
|
||||
|
||||
// esclient.search( query, ( err, res ) => {
|
||||
// console.error( err, res );
|
||||
// });
|
||||
}; |
||||
|
||||
module.exports = TypeMapping; |
||||
@ -1,86 +1,8 @@
|
||||
const _ = require('lodash'); |
||||
const TypeMapping = require('./TypeMapping'); |
||||
|
||||
function addStandardTargetsToAliases(standard, aliases) { |
||||
var combined = _.extend({}, aliases); |
||||
standard.forEach(function(target) { |
||||
if (combined[target] === undefined) { |
||||
combined[target] = [target]; |
||||
} |
||||
}); |
||||
// instantiate a new type mapping
|
||||
var tm = new TypeMapping(); |
||||
tm.load(); |
||||
|
||||
return combined; |
||||
} |
||||
|
||||
/* |
||||
* Sources |
||||
*/ |
||||
|
||||
// a list of all sources
|
||||
var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'whosonfirst']; |
||||
|
||||
/* |
||||
* A list of alternate names for sources, mostly used to save typing |
||||
*/ |
||||
var SOURCE_ALIASES = { |
||||
'osm': ['openstreetmap'], |
||||
'oa': ['openaddresses'], |
||||
'gn': ['geonames'], |
||||
'wof': ['whosonfirst'] |
||||
}; |
||||
|
||||
/* |
||||
* Create an object that contains all sources or aliases. The key is the source or alias, |
||||
* the value is either that source, or the canonical name for that alias if it's an alias. |
||||
*/ |
||||
var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES); |
||||
|
||||
/* |
||||
* Layers |
||||
*/ |
||||
|
||||
/* |
||||
* A list of all layers in each source. This is used for convenience elswhere |
||||
* and to determine when a combination of source and layer parameters is |
||||
* not going to match any records and will return no results. |
||||
*/ |
||||
var LAYERS_BY_SOURCE = { |
||||
openstreetmap: [ 'address', 'venue', 'street' ], |
||||
openaddresses: [ 'address' ], |
||||
geonames: [ 'country','macroregion', 'region', 'county','localadmin', |
||||
'locality','borough', 'neighbourhood', 'venue' ], |
||||
whosonfirst: [ 'continent', 'empire', 'country', 'dependency', 'macroregion', 'region', |
||||
'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', |
||||
'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode', |
||||
'continent', 'ocean', 'marinearea'] |
||||
}; |
||||
|
||||
/* |
||||
* A list of layer aliases that can be used to support specific use cases |
||||
* (like coarse geocoding) * or work around the fact that different sources |
||||
* may have layers that mean the same thing but have a different name |
||||
*/ |
||||
var LAYER_ALIASES = { |
||||
'coarse': [ 'continent', 'empire', 'country', 'dependency', 'macroregion', |
||||
'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', |
||||
'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode', |
||||
'continent', 'ocean', 'marinearea'] |
||||
}; |
||||
|
||||
// create a list of all layers by combining each entry from LAYERS_BY_SOURCE
|
||||
var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) { |
||||
return acc.concat(LAYERS_BY_SOURCE[key]); |
||||
}, [])); |
||||
|
||||
/* |
||||
* Create the an object that has a key for each possible layer or alias, |
||||
* and returns either that layer, or all the layers in the alias |
||||
*/ |
||||
var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES); |
||||
|
||||
module.exports = { |
||||
sources: SOURCES, |
||||
layers: LAYERS, |
||||
source_mapping: SOURCE_MAPPING, |
||||
layer_mapping: LAYER_MAPPING, |
||||
layers_by_source: LAYERS_BY_SOURCE |
||||
}; |
||||
// export singleton
|
||||
module.exports = tm; |
||||
@ -0,0 +1,208 @@
|
||||
const _ = require('lodash'); |
||||
const TypeMapping = require('../../../helper/TypeMapping'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.interface = function(test) { |
||||
test('valid interface', function(t) { |
||||
t.equal(typeof TypeMapping, 'function', 'TypeMapping is a function'); |
||||
|
||||
t.equal(typeof TypeMapping.addStandardTargetsToAliases, 'function', 'addStandardTargetsToAliases() is a function'); |
||||
t.equal(typeof TypeMapping.prototype.setSourceAliases, 'function', 'setSourceAliases() is a function'); |
||||
|
||||
t.equal(typeof TypeMapping.prototype.setLayersBySource, 'function', 'setLayersBySource() is a function'); |
||||
t.equal(typeof TypeMapping.prototype.setLayerAliases, 'function', 'setLayerAliases() is a function'); |
||||
|
||||
t.equal(typeof TypeMapping.prototype.generateMappings, 'function', 'generateMappings() is a function'); |
||||
|
||||
t.equal(typeof TypeMapping.prototype.loadTargets, 'function', 'loadTargets() is a function'); |
||||
t.equal(typeof TypeMapping.prototype.load, 'function', 'load() is a function'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.constructor = function(test) { |
||||
test('constructor', function(t) { |
||||
|
||||
var doc = new TypeMapping(); |
||||
|
||||
// initial values
|
||||
t.deepEqual(doc.sources, [], 'initial value'); |
||||
t.deepEqual(doc.source_aliases, {}, 'initial value'); |
||||
t.deepEqual(doc.layers, [], 'initial value'); |
||||
t.deepEqual(doc.layers_by_source, {}, 'initial value'); |
||||
t.deepEqual(doc.layer_aliases, {}, 'initial value'); |
||||
t.deepEqual(doc.source_mapping, {}, 'initial value'); |
||||
t.deepEqual(doc.layer_mapping, {}, 'initial value'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.addStandardTargetsToAliases = function(test) { |
||||
test('static method addStandardTargetsToAliases', function(t) { |
||||
|
||||
var aliases = { test: ['test2'] }; |
||||
|
||||
t.deepEqual( |
||||
TypeMapping.addStandardTargetsToAliases([], aliases), |
||||
{ test: ['test2'] } |
||||
); |
||||
t.deepEqual(aliases, aliases, 'aliases object not mutated'); |
||||
|
||||
t.deepEqual( |
||||
TypeMapping.addStandardTargetsToAliases(['test'], aliases), |
||||
{ test: ['test2'] }, |
||||
'not modified' |
||||
); |
||||
t.deepEqual(aliases, aliases, 'aliases object not mutated'); |
||||
|
||||
t.deepEqual( |
||||
TypeMapping.addStandardTargetsToAliases(['baz'], aliases), |
||||
{ test: ['test2'], baz: ['baz'] } |
||||
); |
||||
t.deepEqual(aliases, aliases, 'aliases object not mutated'); |
||||
|
||||
t.deepEqual( |
||||
TypeMapping.addStandardTargetsToAliases(['baz','boo'], aliases), |
||||
{ test: ['test2'], baz: ['baz'], boo: ['boo'] } |
||||
); |
||||
t.deepEqual(aliases, aliases, 'aliases object not mutated'); |
||||
|
||||
t.end(); |
||||
|
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.setSourceAliases = function(test) { |
||||
test('setter setSourceAliases', function(t) { |
||||
var tm = new TypeMapping(); |
||||
t.deepEqual(tm.source_aliases, {}); |
||||
tm.setSourceAliases({ foo: ['foo', 'bar'] }); |
||||
t.deepEqual(tm.source_aliases, { foo: ['foo', 'bar'] }); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.setLayersBySource = function(test) { |
||||
test('setter setLayersBySource', function(t) { |
||||
var tm = new TypeMapping(); |
||||
t.deepEqual(tm.layers_by_source, {}); |
||||
tm.setLayersBySource({ foo: ['foo', 'bar'] }); |
||||
t.deepEqual(tm.layers_by_source, { foo: ['foo', 'bar'] }); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.setLayerAliases = function(test) { |
||||
test('setter setLayerAliases', function(t) { |
||||
var tm = new TypeMapping(); |
||||
t.deepEqual(tm.layer_aliases, {}); |
||||
tm.setLayerAliases({ foo: ['foo', 'bar'] }); |
||||
t.deepEqual(tm.layer_aliases, { foo: ['foo', 'bar'] }); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.generateMappings = function(test) { |
||||
test('generateMappings - no-op', function(t) { |
||||
var tm = new TypeMapping(); |
||||
t.deepEqual(tm.sources, []); |
||||
t.deepEqual(tm.source_mapping, {}); |
||||
t.deepEqual(tm.layers, []); |
||||
t.deepEqual(tm.layer_mapping, {}); |
||||
tm.generateMappings(); |
||||
t.deepEqual(tm.sources, []); |
||||
t.deepEqual(tm.source_mapping, {}); |
||||
t.deepEqual(tm.layers, []); |
||||
t.deepEqual(tm.layer_mapping, {}); |
||||
t.end(); |
||||
}); |
||||
test('generateMappings - sources', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; |
||||
tm.generateMappings(); |
||||
t.deepEqual(tm.sources, ['foo', 'faz']); |
||||
t.end(); |
||||
}); |
||||
test('generateMappings - source_mapping', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; |
||||
tm.source_aliases = { foo: ['foo','f'], bar: ['bar', 'b'], baz: ['baz'] }; |
||||
tm.generateMappings(); |
||||
t.deepEqual(tm.source_mapping, { foo: ['foo', 'f'], bar: ['bar', 'b'], baz: ['baz'], faz: ['faz'] }); |
||||
t.end(); |
||||
}); |
||||
test('generateMappings - layers', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; |
||||
tm.generateMappings(); |
||||
t.deepEqual(tm.layers, ['foo','faz']); |
||||
t.end(); |
||||
}); |
||||
test('generateMappings - layer_mapping', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; |
||||
tm.layer_aliases = { foo: ['foo','f'], bar: ['bar', 'b'], baz: ['baz'] }; |
||||
tm.generateMappings(); |
||||
t.deepEqual(tm.layer_mapping, { foo: ['foo', 'f'], bar: ['bar', 'b'], baz: ['baz'], faz: ['faz'] }); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.loadTargets = function(test) { |
||||
test('loadTargets - undefined', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.loadTargets(); |
||||
t.deepEqual(tm.sources, []); |
||||
t.deepEqual(tm.source_mapping, {}); |
||||
t.deepEqual(tm.layers, []); |
||||
t.deepEqual(tm.layer_mapping, {}); |
||||
t.end(); |
||||
}); |
||||
test('loadTargets', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.loadTargets({ |
||||
source_aliases: { source1: ['s1', 's2'], source2: ['s3', 's4'] }, |
||||
layers_by_source: { source1: ['layer1', 'layer3'], source2: ['layer2'] }, |
||||
layer_aliases: { layer1: ['l1', 'l2'], layer2: ['l3', 'l4'] }, |
||||
}); |
||||
t.deepEqual(tm.sources, [ 'source1', 'source2' ]); |
||||
t.deepEqual(tm.source_mapping, { source1: [ 's1', 's2' ], source2: [ 's3', 's4' ] }); |
||||
t.deepEqual(tm.layers, [ 'layer1', 'layer3', 'layer2' ]); |
||||
t.deepEqual(tm.layer_mapping, { layer1: [ 'l1', 'l2' ], layer2: [ 'l3', 'l4' ], layer3: [ 'layer3' ] }); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.load = function(test) { |
||||
test('load from pelias config', function(t) { |
||||
var tm = new TypeMapping(); |
||||
tm.load(() => { |
||||
|
||||
// load pelias config
|
||||
const expected = _.get( |
||||
require('pelias-config').generate(require('../../../schema')), |
||||
'api.targets', {} |
||||
); |
||||
|
||||
// values copied from config
|
||||
t.deepEqual(tm.layers_by_source, expected.layers_by_source || {}); |
||||
t.deepEqual(tm.source_aliases, expected.source_aliases || {}); |
||||
t.deepEqual(tm.layer_aliases, expected.layer_aliases || {}); |
||||
t.end(); |
||||
}); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
|
||||
function test(name, testFunction) { |
||||
return tape('TypeMapping: ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue