feat(targets): configuration-based filter targets

helper/type_mapping.js

@@ -1,14 +1,42 @@
 const _ = require('lodash');
+const elasticsearch = require('elasticsearch');
 
 var TypeMapping = function(){
+
+  // A list of all sources
   this.sources = [];
+
+  // A list of alternate names for sources, mostly used to save typing
   this.source_aliases = {};
+
+  // A list of all layers
   this.layers = [];
+
+  /*
+   * A list of all layers in each source. This is used for convenience elswhere
+   * and to determine when a combination of source and layer parameters is
+   * not going to match any records and will return no results.
+   */
   this.layers_by_source = {};
+
+  /*
+   * A list of layer aliases that can be used to support specific use cases
+   * (like coarse geocoding) * or work around the fact that different sources
+   * may have layers that mean the same thing but have a different name
+   */
   this.layer_aliases = {};
+
+  /*
+   * An object that contains all sources or aliases. The key is the source or alias,
+   * the value is either that source, or the canonical name for that alias if it's an alias.
+   */
   this.source_mapping = {};
+
+  /*
+   * An object that has a key for each possible layer or alias,
+   * and returns either that layer, or gall the layers in the alias
+   */
   this.layer_mapping = {};
-  this.layers_by_source = {};
 };
 
 TypeMapping.addStandardTargetsToAliases = function(standard, aliases) {
@@ -22,76 +50,94 @@ TypeMapping.addStandardTargetsToAliases = function(standard, aliases) {
   return combined;
 };
 
-TypeMapping.prototype.setSources = function( sources ){ this.sources = sources; };
+// source alias setter
-TypeMapping.prototype.setSourceAliases = function( aliases ){ this.source_aliases = aliases; };
+TypeMapping.prototype.setSourceAliases = function( aliases ){
-TypeMapping.prototype.setLayersBySource = function( lbs ){ this.layers_by_source = lbs; };
+  this.source_aliases = aliases;
-TypeMapping.prototype.setLayerAliases = function( aliases ){ this.layer_aliases = aliases; };
+};
-TypeMapping.prototype.generateDynamicMappings = function(){
 
-  /*
+// layers-by-source alias setter
-   * Create an object that contains all sources or aliases. The key is the source or alias,
+TypeMapping.prototype.setLayersBySource = function( lbs ){
-   * the value is either that source, or the canonical name for that alias if it's an alias.
+  this.layers_by_source = lbs;
-   */
+};
-  this.source_mapping = TypeMapping.addStandardTargetsToAliases(this.sources, this.source_aliases);
 
-  // create a list of all layers by combining each entry from this.layers_by_source
+// layer alias setter
+TypeMapping.prototype.setLayerAliases = function( aliases ){
+  this.layer_aliases = aliases;
+};
+
+// generate dynamic mappings after setters have been run
+TypeMapping.prototype.generateDynamicMappings = function(){
+  this.sources = Object.keys( this.layers_by_source );
+  this.source_mapping = TypeMapping.addStandardTargetsToAliases(this.sources, this.source_aliases);
   this.layers = _.uniq(Object.keys(this.layers_by_source).reduce(function(acc, key) {
     return acc.concat(this.layers_by_source[key]);
   }.bind(this), []));
-
-  /*
-   * Create the an object that has a key for each possible layer or alias,
-   * and returns either that layer, or all the layers in the alias
-   */
   this.layer_mapping = TypeMapping.addStandardTargetsToAliases(this.layers, this.layer_aliases);
 };
 
-// instantiate a new type mapping
+// load values from targets block
-var tm = new TypeMapping();
+TypeMapping.prototype.loadTargets = function( targetsBlock ){
 
-// a list of all sources
+  // set values from targets block
-tm.setSources([ 'openstreetmap', 'openaddresses', 'geonames', 'whosonfirst' ]);
+  this.setSourceAliases( targetsBlock.source_aliases || {} );
+  this.setLayersBySource( targetsBlock.layers_by_source || {} );
+  this.setLayerAliases( targetsBlock.layer_aliases || {} );
 
-/*
+  // generate the dynamic mappings
- * A list of alternate names for sources, mostly used to save typing
+  this.generateDynamicMappings();
- */
+};
-tm.setSourceAliases({
-  'osm': [ 'openstreetmap' ],
-  'oa':  [ 'openaddresses' ],
-  'gn':  [ 'geonames' ],
-  'wof': [ 'whosonfirst' ]
-});
 
-/*
+// load values from either pelias config file or from elasticsearch
- * A list of all layers in each source. This is used for convenience elswhere
+TypeMapping.prototype.load = function( done ){
- * and to determine when a combination of source and layer parameters is
- * not going to match any records and will return no results.
- */
-tm.setLayersBySource({
-  openstreetmap: [ 'address', 'venue', 'street' ],
-  openaddresses: [ 'address' ],
-  geonames: [ 'country','macroregion', 'region', 'county','localadmin',
-    'locality','borough', 'neighbourhood', 'venue' ],
-  whosonfirst: [ 'continent', 'empire', 'country', 'dependency', 'macroregion',
-    'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 
-    'borough', 'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode', 
-    'continent', 'ocean', 'marinearea' ]
-});
 
-/*
+  // load pelias config
- * A list of layer aliases that can be used to support specific use cases
+  const peliasConfig = require('pelias-config').generate(require('../schema'));
- * (like coarse geocoding) * or work around the fact that different sources
- * may have layers that mean the same thing but have a different name
- */
-tm.setLayerAliases({
-  'coarse': [ 'continent', 'empire', 'country', 'dependency', 'macroregion',
-    'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood',
-    'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode',
-    'continent', 'ocean', 'marinearea' ]
-});
 
-// generate the dynamic mappings
+  // load targets from config file
-tm.generateDynamicMappings();
+  this.loadTargets( peliasConfig.api.targets );
+
+  // do not load values from elasticsearch
+  if( true !== peliasConfig.api.targets.auto_discover ){
+    if( 'function' === typeof done ){ done(); }
+    return;
+  }
+
+  // load values from elasticsearch
+
+  // create connection to elasticsearch
+  // const esclient = elasticsearch.Client(peliasConfig.esclient);
+
+  // const query = {
+  //   requestCache: true,
+  //   preference: '_replica_first',
+  //   timeout: '10s',
+  //   body: {
+  //     aggs: {
+  //       sources: {
+  //         terms: {
+  //           field: 'source',
+  //           size: 100
+  //         }
+  //       },
+  //       layers: {
+  //         terms: {
+  //           field: 'layer',
+  //           size: 100
+  //         }
+  //       }
+  //     },
+  //     size: 0
+  //   }
+  // };
+
+  // esclient.search( query, ( err, res ) => {
+  //   console.error( err, res );
+  // });
+};
+
+// instantiate a new type mapping
+var tm = new TypeMapping();
+tm.load();
 
 // export singleton
 module.exports = tm;

index.js

@@ -1,10 +1,14 @@
 
 const app = require('./app'),
+    typeMapping = require('./helper/type_mapping'),
     port = ( process.env.PORT || 3100 ),
     host = ( process.env.HOST || undefined );
 
+// ensure the dynamic type mappings are loaded before starting the server.
+typeMapping.load(() => {
   const server = app.listen( port, host, () => {
     // ask server for the actual address and port its listening on
     const listenAddress = server.address();
     console.log( `pelias is now running on ${listenAddress.address}:${listenAddress.port}` );
   });
+});

package.json

@@ -53,7 +53,7 @@
     "markdown": "0.5.0",
     "morgan": "^1.8.2",
     "pelias-categories": "1.2.0",
-    "pelias-config": "2.14.0",
+    "pelias-config": "git://github.com/pelias/config.git#config_source_layers",
     "pelias-labels": "1.8.0",
     "pelias-logger": "0.3.1",
     "pelias-microservice-wrapper": "1.3.0",