Browse Source

feat(dedupe): move canonical_sources config to pelias/config

pull/1222/head
missinglink 6 years ago committed by Julian Simioni
parent
commit
077bd3f78e
No known key found for this signature in database
GPG Key ID: B9EEB0C6EE0910A1
  1. 21
      helper/TypeMapping.js
  2. 8
      middleware/dedupe.js
  3. 2
      package.json

21
helper/TypeMapping.js

@ -1,9 +1,6 @@
const _ = require('lodash'); const _ = require('lodash');
const elasticsearch = require('elasticsearch'); const elasticsearch = require('elasticsearch');
// a list of the canonical sources included in the default Pelias configuration
const CANONICAL_SOURCES = ['whosonfirst', 'openstreetmap', 'openaddresses', 'geonames'];
var TypeMapping = function(){ var TypeMapping = function(){
// A list of all sources // A list of all sources
@ -29,6 +26,11 @@ var TypeMapping = function(){
*/ */
this.layer_aliases = {}; this.layer_aliases = {};
/*
* A list of the canonical sources included in the default Pelias configuration
*/
this.canonical_sources = [];
/* /*
* An object that contains all sources or aliases. The key is the source or alias, * An object that contains all sources or aliases. The key is the source or alias,
* the value is either that source, or the canonical name for that alias if it's an alias. * the value is either that source, or the canonical name for that alias if it's an alias.
@ -68,6 +70,11 @@ TypeMapping.prototype.setLayerAliases = function( aliases ){
this.layer_aliases = aliases; this.layer_aliases = aliases;
}; };
// canonical sources setter
TypeMapping.prototype.setCanonicalSources = function( sources ){
this.canonical_sources = sources;
};
// generate mappings after setters have been run // generate mappings after setters have been run
TypeMapping.prototype.generateMappings = function(){ TypeMapping.prototype.generateMappings = function(){
this.sources = Object.keys( this.layers_by_source ); this.sources = Object.keys( this.layers_by_source );
@ -78,16 +85,11 @@ TypeMapping.prototype.generateMappings = function(){
this.layer_mapping = TypeMapping.addStandardTargetsToAliases(this.layers, this.layer_aliases); this.layer_mapping = TypeMapping.addStandardTargetsToAliases(this.layers, this.layer_aliases);
}; };
// return a list of all sources which are part of the canonical Pelias configuration
TypeMapping.prototype.getCanonicalSources = function(){
return CANONICAL_SOURCES;
};
// generate a list of all layers which are part of the canonical Pelias configuration // generate a list of all layers which are part of the canonical Pelias configuration
TypeMapping.prototype.getCanonicalLayers = function(){ TypeMapping.prototype.getCanonicalLayers = function(){
var canonicalLayers = []; var canonicalLayers = [];
for( var source in this.layers_by_source ){ for( var source in this.layers_by_source ){
if( _.includes( CANONICAL_SOURCES, source ) ){ if( _.includes( this.canonical_sources, source ) ){
canonicalLayers = _.uniq( canonicalLayers.concat( this.layers_by_source[source] ) ); canonicalLayers = _.uniq( canonicalLayers.concat( this.layers_by_source[source] ) );
} }
} }
@ -103,6 +105,7 @@ TypeMapping.prototype.loadTargets = function( targetsBlock ){
this.setSourceAliases( targetsBlock.source_aliases || {} ); this.setSourceAliases( targetsBlock.source_aliases || {} );
this.setLayersBySource( targetsBlock.layers_by_source || {} ); this.setLayersBySource( targetsBlock.layers_by_source || {} );
this.setLayerAliases( targetsBlock.layer_aliases || {} ); this.setLayerAliases( targetsBlock.layer_aliases || {} );
this.setCanonicalSources( targetsBlock.canonical_sources || [] );
// generate the mappings // generate the mappings
this.generateMappings(); this.generateMappings();

8
middleware/dedupe.js

@ -1,7 +1,7 @@
const logger = require('pelias-logger').get('api'); const logger = require('pelias-logger').get('api');
const _ = require('lodash'); const _ = require('lodash');
const isDifferent = require('../helper/diffPlaces').isDifferent; const isDifferent = require('../helper/diffPlaces').isDifferent;
const canonicalSources = require('../helper/type_mapping').getCanonicalSources(); const canonical_sources = require('../helper/type_mapping').canonical_sources;
const field = require('../helper/fieldValue'); const field = require('../helper/fieldValue');
function dedupeResults(req, res, next) { function dedupeResults(req, res, next) {
@ -13,7 +13,7 @@ function dedupeResults(req, res, next) {
if( _.isUndefined(res) || !_.isArray(res.data) || _.isEmpty(res.data) ){ return next(); } if( _.isUndefined(res) || !_.isArray(res.data) || _.isEmpty(res.data) ){ return next(); }
// loop through data items and only copy unique items to unique // loop through data items and only copy unique items to unique
// note: the first reqults must always be unique! // note: the first results must always be unique!
let unique = [ res.data[0] ]; let unique = [ res.data[0] ];
// convenience function to search unique array for an existing element which matches a hit // convenience function to search unique array for an existing element which matches a hit
@ -76,8 +76,8 @@ function isPreferred(existingHit, candidateHit) {
_.has(candidateHit, 'address_parts.zip') ){ return true; } _.has(candidateHit, 'address_parts.zip') ){ return true; }
// prefer non-canonical sources over canonical ones // prefer non-canonical sources over canonical ones
if( !_.includes(canonicalSources, candidateHit.source) && if( !_.includes(canonical_sources, candidateHit.source) &&
_.includes(canonicalSources, existingHit.source) ){ return true; } _.includes(canonical_sources, existingHit.source) ){ return true; }
// prefer certain sources over others // prefer certain sources over others
switch( existingHit.source ){ switch( existingHit.source ){

2
package.json

@ -52,7 +52,7 @@
"markdown": "^0.5.0", "markdown": "^0.5.0",
"morgan": "^1.8.2", "morgan": "^1.8.2",
"pelias-categories": "^1.2.0", "pelias-categories": "^1.2.0",
"pelias-config": "^3.0.2", "pelias-config": "^3.7.0",
"pelias-labels": "^1.8.0", "pelias-labels": "^1.8.0",
"pelias-logger": "^1.2.0", "pelias-logger": "^1.2.0",
"pelias-microservice-wrapper": "^1.7.0", "pelias-microservice-wrapper": "^1.7.0",

Loading…
Cancel
Save