Merge pull request #597 from pelias/master

Merge master into staging

.travis.yml

@@ -1,23 +1,30 @@
 sudo: false
 language: node_js
+cache:
+  directories:
+    - node_modules
+notifications:
+  email: false
 node_js:
-  - 0.10
   - 0.12
-  - 4.4
+  - 4
-  - 5.8
+  - 5
+  - 6
 matrix:
+    fast_finish: true
     allow_failures:
-        - node_js: 4.4
+        - node_js: 6
-        - node_js: 5.8
 env:
   global:
     - CXX=g++-4.8
-  matrix:
+script: "npm run travis"
-    - TEST_SUITE=unit
-script: "npm run $TEST_SUITE"
 addons:
   apt:
     sources:
       - ubuntu-toolchain-r-test
     packages:
       - g++-4.8
+before_install:
+  - npm i -g npm@^2.0.0
+before_script:
+  - npm prune

controller/place.js

@@ -1,7 +1,7 @@
 var service = { mget: require('../service/mget') };
 var logger = require('pelias-logger').get('api:controller:place');
 
-function setup( backend ){
+function setup( config, backend ){
 
   // allow overriding of dependencies
   backend = backend || require('../src/backend');
@@ -16,7 +16,7 @@ function setup( backend ){
 
     var query = req.clean.ids.map( function(id) {
       return {
-        _index: 'pelias',
+        _index: config.indexName,
         _type: id.layers,
         _id: id.id
       };

controller/search.js

@@ -4,7 +4,7 @@ var service = { search: require('../service/search') };
 var logger = require('pelias-logger').get('api:controller:search');
 var logging = require( '../helper/logging' );
 
-function setup( backend, query ){
+function setup( config, backend, query ){
 
   // allow overriding of dependencies
   backend = backend || require('../src/backend');
@@ -26,16 +26,11 @@ function setup( backend, query ){
 
     // backend command
     var cmd = {
-      index: 'pelias',
+      index: config.indexName,
       searchType: 'dfs_query_then_fetch',
       body: query( req.clean )
     };
 
-    // use layers field for filtering by type
-    if( req.clean.hasOwnProperty('layers') ){
-      cmd.type = req.clean.layers;
-    }
-
     logger.debug( '[ES req]', cmd );
 
     // query backend

helper/labelSchema.js

@@ -12,7 +12,7 @@ module.exports = {
   },
   'USA': {
     'borough': getFirstProperty(['borough']),
-    'local': getFirstProperty(['locality', 'localadmin']),
+    'local': getFirstProperty(['locality', 'localadmin', 'county']),
     'regional': getUsOrCaState,
     'country': getUSACountryValue
   },

index.js

@@ -1,20 +1,32 @@
 
-var Cluster = require('cluster2'),
+var cluster = require('cluster'),
     app = require('./app'),
     port = ( process.env.PORT || 3100 ),
     multicore = true;
 
 /** cluster webserver across all cores **/
 if( multicore ){
-  var c = new Cluster({ port: port });
+
-  c.listen(function(cb){
+  var numCPUs = require('os').cpus().length;
+  if( cluster.isMaster ){
+
+    // fork workers
+    for (var i = 0; i < numCPUs; i++) {
+      cluster.fork();
+    }
+
+    cluster.on('exit', function( worker, code, signal ){
+      console.log('worker ' + worker.process.pid + ' died');
+    });
+
+  } else {
+    app.listen( port );
     console.log( 'worker: listening on ' + port );
-    cb(app);
+  }
-  });
 }
 
 /** run server on the default setup (single core) **/
 else {
   console.log( 'listening on ' + port );
   app.listen( port );
-}
+}

middleware/localNamingConventions.js

@@ -1,7 +1,7 @@
 var check = require('check-types');
 var _ = require('lodash');
 
-var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL'];
+var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL', 'CZE'];
 
 function setup() {
   var api = require('pelias-config').generate().api;

package.json

@@ -7,14 +7,15 @@
   "license": "MIT",
   "main": "index.js",
   "scripts": {
-    "start": "node index.js",
+    "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
-    "test": "npm run unit",
-    "unit": "./bin/units",
     "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao",
     "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
-    "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
     "docs": "./bin/generate-docs",
     "lint": "jshint .",
+    "start": "node index.js",
+    "test": "npm run unit",
+    "travis": "npm test",
+    "unit": "./bin/units",
     "validate": "npm ls"
   },
   "repository": {
@@ -33,9 +34,8 @@
     "node": ">=0.10.26"
   },
   "dependencies": {
-    "async": "^1.5.2",
+    "async": "^2.0.0",
     "check-types": "^7.0.0",
-    "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",
     "elasticsearch": "^11.0.0",
     "elasticsearch-exceptions": "0.0.4",
     "express": "^4.8.8",
@@ -49,13 +49,12 @@
     "lodash": "^4.5.0",
     "markdown": "0.5.0",
     "morgan": "1.7.0",
-    "pelias-config": "~2.0.0",
+    "pelias-config": "2.1.0",
-    "pelias-logger": "^0.0.8",
+    "pelias-logger": "0.0.8",
-    "pelias-model": "^4.0.0",
+    "pelias-model": "4.1.0",
-    "pelias-query": "~8.0.0",
+    "pelias-query": "8.1.3",
-    "pelias-suggester-pipeline": "2.0.4",
+    "pelias-text-analyzer": "1.1.0",
-    "pelias-text-analyzer": "^1.0.1",
+    "stats-lite": "2.0.3",
-    "stats-lite": "1.0.3",
     "through2": "2.0.1"
   },
   "devDependencies": {

query/search.js

@@ -43,6 +43,7 @@ query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') );
 query.filter( peliasQuery.view.boundary_circle );
 query.filter( peliasQuery.view.boundary_rect );
 query.filter( peliasQuery.view.sources );
+query.filter( peliasQuery.view.layers );
 // --------------------------------
 
 /**
@@ -59,6 +60,9 @@ function generateQuery( clean ){
   // sources
   vs.var( 'sources', clean.sources);
 
+  // layers
+  vs.var( 'layers', clean.layers);
+
   // size
   if( clean.querySize ) {
     vs.var( 'size', clean.querySize );

routes/v1.js

@@ -61,7 +61,7 @@ function addRoutes(app, peliasConfig) {
     search: createRouter([
       sanitisers.search.middleware,
       middleware.calcSize(),
-      controllers.search(),
+      controllers.search(peliasConfig),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),
@@ -74,7 +74,7 @@ function addRoutes(app, peliasConfig) {
     ]),
     autocomplete: createRouter([
       sanitisers.autocomplete.middleware,
-      controllers.search(null, require('../query/autocomplete')),
+      controllers.search(peliasConfig, null, require('../query/autocomplete')),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),
@@ -88,7 +88,7 @@ function addRoutes(app, peliasConfig) {
     reverse: createRouter([
       sanitisers.reverse.middleware,
       middleware.calcSize(),
-      controllers.search(undefined, reverseQuery),
+      controllers.search(peliasConfig, undefined, reverseQuery),
       postProc.distances('point.'),
       // reverse confidence scoring depends on distance from origin
       //  so it must be calculated first
@@ -103,7 +103,7 @@ function addRoutes(app, peliasConfig) {
     ]),
     place: createRouter([
       sanitisers.place.middleware,
-      controllers.place(),
+      controllers.place(peliasConfig),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),

sanitiser/_ids.js

@@ -34,9 +34,9 @@ function sanitizeId(rawId, messages) {
     messages.errors.push( formatError(rawId) );
     return;
   }
-
+  var valid_values = Object.keys(type_mapping.source_mapping);
-  if (!_.includes(type_mapping.sources, source)) {
+  if (!_.includes(valid_values, source)) {
-    messages.errors.push( targetError(source, type_mapping.sources) );
+    messages.errors.push( targetError(source, valid_values) );
     return;
   }
 
@@ -46,7 +46,7 @@ function sanitizeId(rawId, messages) {
   }
 
   return {
-    source: source,
+    source: type_mapping.source_mapping[source][0],
     layer: layer,
     id: id,
   };

test/ciao_test_data.js

@@ -15,6 +15,7 @@
 var client = require('elasticsearch').Client(),
     async = require('async'),
     actions = [];
+var config = require('pelias-config').generate().api;
 
 // add one record per 'type' in order to cause the _default_ mapping
 // to be copied when the new type is created.
@@ -28,7 +29,7 @@ types.forEach( function( type, i1 ){
     layers.forEach( function( layer, i3 ){
       actions.push( function( done ){
         client.index({
-          index: 'pelias',
+          index: config.indexName,
           type: type,
           id: [i1,i2,i3].join(':'),
           body: {
@@ -49,7 +50,7 @@ types.forEach( function( type, i1 ){
 
 // call refresh so the index merges the changes
 actions.push( function( done ){
-  client.indices.refresh( { index: 'pelias' }, done);
+  client.indices.refresh( { index: config.indexName }, done);
 });
 
 // perform all actions in series

test/unit/controller/place.js

@@ -11,6 +11,11 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
+// reminder: this is only the api subsection of the full config
+var fakeDefaultConfig = {
+  indexName: 'pelias'
+};
+
 // functionally test controller (backend success)
 module.exports.tests.functional_success = function(test, common) {
 
@@ -43,7 +48,37 @@ module.exports.tests.functional_success = function(test, common) {
     var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
       t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command');
     });
-    var controller = setup( backend );
+    var controller = setup( fakeDefaultConfig, backend );
+    var res = {
+      status: function( code ){
+        t.equal(code, 200, 'status set');
+        return res;
+      },
+      json: function( json ){
+        t.equal(typeof json, 'object', 'returns json');
+        t.equal(typeof json.date, 'number', 'date set');
+        t.equal(json.type, 'FeatureCollection', 'valid geojson');
+        t.true(Array.isArray(json.features), 'features is array');
+        t.deepEqual(json.features, expected, 'values correctly mapped');
+      }
+    };
+    var req = { clean: { ids: [ {'id' : 123, layers: [ 'a' ] } ] }, errors: [], warnings: [] };
+    var next = function next() {
+      t.equal(req.errors.length, 0, 'next was called without error');
+      t.end();
+    };
+    controller(req, res, next );
+  });
+
+  test('functional success with custom index name', function(t) {
+    var fakeCustomizedConfig = {
+      indexName: 'alternateindexname'
+    };
+
+    var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
+      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'alternateindexname', _type: [ 'a' ] } ] } }, 'correct backend command');
+    });
+    var controller = setup( fakeCustomizedConfig, backend );
     var res = {
       status: function( code ){
         t.equal(code, 200, 'status set');
@@ -72,7 +107,7 @@ module.exports.tests.functional_failure = function(test, common) {
     var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
       t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: ['b'] } ] } }, 'correct backend command');
     });
-    var controller = setup( backend );
+    var controller = setup( fakeDefaultConfig, backend );
     var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] };
     var next = function( message ){
       t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler');

test/unit/controller/search.js

@@ -1,4 +1,3 @@
-
 var setup = require('../../../controller/search'),
     mockBackend = require('../mock/backend'),
     mockQuery = require('../mock/query');
@@ -13,6 +12,11 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
+// reminder: this is only the api subsection of the full config
+var fakeDefaultConfig = {
+  indexName: 'pelias'
+};
+
 // functionally test controller (backend success)
 module.exports.tests.functional_success = function(test, common) {
 
@@ -82,7 +86,7 @@ module.exports.tests.functional_success = function(test, common) {
         searchType: 'dfs_query_then_fetch'
       }, 'correct backend command');
     });
-    var controller = setup(backend, mockQuery());
+    var controller = setup(fakeDefaultConfig, backend, mockQuery());
     var res = {
       status: function (code) {
         t.equal(code, 200, 'status set');
@@ -105,6 +109,33 @@ module.exports.tests.functional_success = function(test, common) {
     };
     controller(req, res, next);
   });
+
+  test('functional success with alternate index name', function(t) {
+    var fakeCustomizedConfig = {
+      indexName: 'alternateindexname'
+    };
+
+    var backend = mockBackend('client/search/ok/1', function (cmd) {
+      t.deepEqual(cmd, {
+        body: {a: 'b'},
+        index: 'alternateindexname',
+        searchType: 'dfs_query_then_fetch'
+      }, 'correct backend command');
+    });
+    var controller = setup(fakeCustomizedConfig, backend, mockQuery());
+    var res = {
+      status: function (code) {
+        t.equal(code, 200, 'status set');
+        return res;
+      }
+    };
+    var req = { clean: { a: 'b' }, errors: [], warnings: [] };
+    var next = function next() {
+      t.equal(req.errors.length, 0, 'next was called without error');
+      t.end();
+    };
+    controller(req, res, next);
+  });
 };
 
 // functionally test controller (backend failure)
@@ -113,7 +144,7 @@ module.exports.tests.functional_failure = function(test, common) {
     var backend = mockBackend( 'client/search/fail/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( backend, mockQuery() );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'a backend error occurred');
@@ -128,7 +159,7 @@ module.exports.tests.timeout = function(test, common) {
     var backend = mockBackend( 'client/search/timeout/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( backend, mockQuery() );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'Request Timeout after 5000ms');

test/unit/fixture/search_boundary_country.js

@@ -81,7 +81,16 @@ module.exports = {
             'weight': 2
           }]
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
     }
   },
   'sort': [ '_score' ],

test/unit/fixture/search_full_address.js

@@ -128,7 +128,23 @@ module.exports = {
             'query': 'new york',
             'analyzer': 'peliasAdmin'
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'address',
+              'venue',
+              'country',
+              'region',
+              'county',
+              'neighbourhood',
+              'locality',
+              'localadmin'
+            ]
+          }
+        }
+      ]
     }
   },
   'size': 10,

test/unit/fixture/search_linguistic_bbox.js

@@ -74,14 +74,21 @@ module.exports = {
       }],
       'filter': [{
         'geo_bounding_box': {
+          'type': 'indexed',
           'center_point': {
             'top': 11.51,
             'right': -61.84,
             'bottom': 47.47,
             'left': -103.16
-          },
+            }
-          'type': 'indexed'
+          }
-        }
+        },
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
       }]
     }
   },

test/unit/fixture/search_linguistic_focus.js

@@ -101,7 +101,16 @@ module.exports = {
             'weight': 2
           }]
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
     }
   },
   'sort': [ '_score' ],

test/unit/fixture/search_linguistic_focus_bbox.js

@@ -104,13 +104,20 @@ module.exports = {
       }],
       'filter': [{
         'geo_bounding_box': {
+          'type': 'indexed',
           'center_point': {
             'top': 11.51,
             'right': -61.84,
             'bottom': 47.47,
             'left': -103.16
-          },
+          }
-          'type': 'indexed'
+        }
+      },
+      {
+        'terms': {
+          'layer': [
+            'test'
+          ]
         }
       }]
     }

test/unit/fixture/search_linguistic_focus_null_island.js

@@ -101,6 +101,13 @@ module.exports = {
             'weight': 2
           }]
         }
+      }],
+      'filter':[{
+        'terms': {
+          'layer': [
+            'test'
+          ]
+        }
       }]
     }
   },

test/unit/fixture/search_linguistic_only.js

@@ -71,7 +71,16 @@ module.exports = {
             'weight': 2
           }]
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
     }
   },
   'sort': [ '_score' ],

test/unit/fixture/search_linguistic_viewport.js

@@ -113,6 +113,15 @@ module.exports = {
             'boost_mode': 'replace'
           }
         }
+      ],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
       ]
     }
   },

test/unit/fixture/search_partial_address.js

@@ -96,7 +96,23 @@ module.exports = {
             'query': 'new york',
             'analyzer': 'peliasAdmin'
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'address',
+              'venue',
+              'country',
+              'region',
+              'county',
+              'neighbourhood',
+              'locality',
+              'localadmin'
+            ]
+          }
+        }
+      ]
     }
   },
   'size': 10,

test/unit/fixture/search_regions_address.js

@@ -112,7 +112,23 @@ module.exports = {
             'query': 'manhattan',
             'analyzer': 'peliasAdmin'
         }
-      }]
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'address',
+              'venue',
+              'country',
+              'region',
+              'county',
+              'neighbourhood',
+              'locality',
+              'localadmin'
+            ]
+          }
+        }
+      ]
     }
   },
   'size': 10,

test/unit/helper/labelGenerator_USA.js

@@ -51,6 +51,25 @@ module.exports.tests.united_states = function(test, common) {
     t.end();
   });
 
+test('county value should be used when there is no localadmin', function(t) {
+    var doc = {
+      'name': 'venue name',
+      'layer': 'venue',
+      'housenumber': 'house number',
+      'street': 'street name',
+      'neighbourhood': 'neighbourhood name',
+      'county': 'county name',
+      'macrocounty': 'macrocounty name',
+      'region_a': 'region abbr',
+      'region': 'region name',
+      'macroregion': 'macroregion name',
+      'country_a': 'USA',
+      'country': 'United States'
+    };
+    t.equal(generator(doc),'venue name, county name, region abbr, USA');
+    t.end();
+  });
+
   test('street', function(t) {
     var doc = {
       'name': 'house number street name',

test/unit/sanitiser/_ids.js

@@ -69,7 +69,8 @@ module.exports.tests.invalid_ids = function(test, common) {
   test('invalid id: source name invalid', function(t) {
     var raw = { ids: 'invalidsource:venue:23' };
     var clean = {};
-    var expected_error = 'invalidsource is invalid. It must be one of these values - [' + type_mapping.sources.join(', ') + ']';
+    var expected_error = 'invalidsource is invalid. It must be one of these values - [' +
+      Object.keys(type_mapping.source_mapping).join(', ') + ']';
 
     var messages = sanitize(raw, clean);
 
@@ -107,6 +108,22 @@ module.exports.tests.valid_ids = function(test, common) {
     t.end();
   });
 
+test('ids: valid short input (openaddresses)', function(t) {
+    var raw = { ids: 'oa:address:20' };
+    var clean = {};
+
+    var messages = sanitize( raw, clean );
+
+    var expected_ids = [{
+      source: 'openaddresses',
+      layer: 'address',
+      id: '20',
+    }];
+    t.deepEqual( messages.errors, [], ' no errors');
+    t.deepEqual( clean.ids, expected_ids, 'single type value returned');
+    t.end();
+  });
+
   test('ids: valid input (osm)', function(t) {
     var raw = { ids: 'openstreetmap:venue:node:500' };
     var clean = {};
@@ -122,6 +139,22 @@ module.exports.tests.valid_ids = function(test, common) {
     t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field');
     t.end();
   });
+
+  test('ids: valid short input (osm)', function(t) {
+    var raw = { ids: 'osm:venue:node:500' };
+    var clean = {};
+    var expected_ids = [{
+      source: 'openstreetmap',
+      layer: 'venue',
+      id: 'node:500',
+    }];
+
+    var messages = sanitize( raw, clean );
+
+    t.deepEqual( messages.errors, [], ' no errors');
+    t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field');
+    t.end();
+  });
 };
 
 module.exports.tests.multiple_ids = function(test, common) {