diff --git a/.travis.yml b/.travis.yml index 0abfae12..e51a8ea7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,30 @@ sudo: false language: node_js +cache: + directories: + - node_modules +notifications: + email: false node_js: - - 0.10 - 0.12 - - 4.4 - - 5.8 + - 4 + - 5 + - 6 matrix: + fast_finish: true allow_failures: - - node_js: 4.4 - - node_js: 5.8 + - node_js: 6 env: global: - CXX=g++-4.8 - matrix: - - TEST_SUITE=unit -script: "npm run $TEST_SUITE" +script: "npm run travis" addons: apt: sources: - ubuntu-toolchain-r-test packages: - g++-4.8 +before_install: + - npm i -g npm@^2.0.0 +before_script: + - npm prune diff --git a/controller/place.js b/controller/place.js index a4996a11..924292b6 100644 --- a/controller/place.js +++ b/controller/place.js @@ -1,7 +1,7 @@ var service = { mget: require('../service/mget') }; var logger = require('pelias-logger').get('api:controller:place'); -function setup( backend ){ +function setup( config, backend ){ // allow overriding of dependencies backend = backend || require('../src/backend'); @@ -16,7 +16,7 @@ function setup( backend ){ var query = req.clean.ids.map( function(id) { return { - _index: 'pelias', + _index: config.indexName, _type: id.layers, _id: id.id }; diff --git a/controller/search.js b/controller/search.js index 7f7ffb68..271d2899 100644 --- a/controller/search.js +++ b/controller/search.js @@ -4,7 +4,7 @@ var service = { search: require('../service/search') }; var logger = require('pelias-logger').get('api:controller:search'); var logging = require( '../helper/logging' ); -function setup( backend, query ){ +function setup( config, backend, query ){ // allow overriding of dependencies backend = backend || require('../src/backend'); @@ -26,16 +26,11 @@ function setup( backend, query ){ // backend command var cmd = { - index: 'pelias', + index: config.indexName, searchType: 'dfs_query_then_fetch', body: query( req.clean ) }; - // use layers field for filtering by type - if( req.clean.hasOwnProperty('layers') ){ - cmd.type = req.clean.layers; - } - logger.debug( '[ES req]', cmd ); // query backend diff --git a/helper/labelSchema.js b/helper/labelSchema.js index 0547e2ad..699e1c88 100644 --- a/helper/labelSchema.js +++ b/helper/labelSchema.js @@ -12,7 +12,7 @@ module.exports = { }, 'USA': { 'borough': getFirstProperty(['borough']), - 'local': getFirstProperty(['locality', 'localadmin']), + 'local': getFirstProperty(['locality', 'localadmin', 'county']), 'regional': getUsOrCaState, 'country': getUSACountryValue }, diff --git a/index.js b/index.js index 5f043b26..8df79add 100644 --- a/index.js +++ b/index.js @@ -1,20 +1,32 @@ -var Cluster = require('cluster2'), +var cluster = require('cluster'), app = require('./app'), port = ( process.env.PORT || 3100 ), multicore = true; /** cluster webserver across all cores **/ if( multicore ){ - var c = new Cluster({ port: port }); - c.listen(function(cb){ + + var numCPUs = require('os').cpus().length; + if( cluster.isMaster ){ + + // fork workers + for (var i = 0; i < numCPUs; i++) { + cluster.fork(); + } + + cluster.on('exit', function( worker, code, signal ){ + console.log('worker ' + worker.process.pid + ' died'); + }); + + } else { + app.listen( port ); console.log( 'worker: listening on ' + port ); - cb(app); - }); + } } /** run server on the default setup (single core) **/ else { console.log( 'listening on ' + port ); app.listen( port ); -} \ No newline at end of file +} diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index e2851370..20c77f6b 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -1,7 +1,7 @@ var check = require('check-types'); var _ = require('lodash'); -var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL']; +var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL', 'CZE']; function setup() { var api = require('pelias-config').generate().api; diff --git a/package.json b/package.json index 1099a2cb..c3a3d01a 100644 --- a/package.json +++ b/package.json @@ -7,14 +7,15 @@ "license": "MIT", "main": "index.js", "scripts": { - "start": "node index.js", - "test": "npm run unit", - "unit": "./bin/units", + "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", - "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", "docs": "./bin/generate-docs", "lint": "jshint .", + "start": "node index.js", + "test": "npm run unit", + "travis": "npm test", + "unit": "./bin/units", "validate": "npm ls" }, "repository": { @@ -33,9 +34,8 @@ "node": ">=0.10.26" }, "dependencies": { - "async": "^1.5.2", + "async": "^2.0.0", "check-types": "^7.0.0", - "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", "elasticsearch": "^11.0.0", "elasticsearch-exceptions": "0.0.4", "express": "^4.8.8", @@ -49,13 +49,12 @@ "lodash": "^4.5.0", "markdown": "0.5.0", "morgan": "1.7.0", - "pelias-config": "~2.0.0", - "pelias-logger": "^0.0.8", - "pelias-model": "^4.0.0", - "pelias-query": "~8.0.0", - "pelias-suggester-pipeline": "2.0.4", - "pelias-text-analyzer": "^1.0.1", - "stats-lite": "1.0.3", + "pelias-config": "2.1.0", + "pelias-logger": "0.0.8", + "pelias-model": "4.1.0", + "pelias-query": "8.1.3", + "pelias-text-analyzer": "1.1.0", + "stats-lite": "2.0.3", "through2": "2.0.1" }, "devDependencies": { diff --git a/query/search.js b/query/search.js index 75406fe7..a6c2fd6a 100644 --- a/query/search.js +++ b/query/search.js @@ -43,6 +43,7 @@ query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') ); query.filter( peliasQuery.view.boundary_circle ); query.filter( peliasQuery.view.boundary_rect ); query.filter( peliasQuery.view.sources ); +query.filter( peliasQuery.view.layers ); // -------------------------------- /** @@ -59,6 +60,9 @@ function generateQuery( clean ){ // sources vs.var( 'sources', clean.sources); + // layers + vs.var( 'layers', clean.layers); + // size if( clean.querySize ) { vs.var( 'size', clean.querySize ); diff --git a/routes/v1.js b/routes/v1.js index c8c88de1..9ef95856 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -61,7 +61,7 @@ function addRoutes(app, peliasConfig) { search: createRouter([ sanitisers.search.middleware, middleware.calcSize(), - controllers.search(), + controllers.search(peliasConfig), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), postProc.dedupe(), @@ -74,7 +74,7 @@ function addRoutes(app, peliasConfig) { ]), autocomplete: createRouter([ sanitisers.autocomplete.middleware, - controllers.search(null, require('../query/autocomplete')), + controllers.search(peliasConfig, null, require('../query/autocomplete')), postProc.distances('focus.point.'), postProc.confidenceScores(peliasConfig), postProc.dedupe(), @@ -88,7 +88,7 @@ function addRoutes(app, peliasConfig) { reverse: createRouter([ sanitisers.reverse.middleware, middleware.calcSize(), - controllers.search(undefined, reverseQuery), + controllers.search(peliasConfig, undefined, reverseQuery), postProc.distances('point.'), // reverse confidence scoring depends on distance from origin // so it must be calculated first @@ -103,7 +103,7 @@ function addRoutes(app, peliasConfig) { ]), place: createRouter([ sanitisers.place.middleware, - controllers.place(), + controllers.place(peliasConfig), postProc.localNamingConventions(), postProc.renamePlacenames(), postProc.parseBoundingBox(), diff --git a/sanitiser/_ids.js b/sanitiser/_ids.js index 4a1844be..ba456766 100644 --- a/sanitiser/_ids.js +++ b/sanitiser/_ids.js @@ -34,9 +34,9 @@ function sanitizeId(rawId, messages) { messages.errors.push( formatError(rawId) ); return; } - - if (!_.includes(type_mapping.sources, source)) { - messages.errors.push( targetError(source, type_mapping.sources) ); + var valid_values = Object.keys(type_mapping.source_mapping); + if (!_.includes(valid_values, source)) { + messages.errors.push( targetError(source, valid_values) ); return; } @@ -46,7 +46,7 @@ function sanitizeId(rawId, messages) { } return { - source: source, + source: type_mapping.source_mapping[source][0], layer: layer, id: id, }; diff --git a/test/ciao_test_data.js b/test/ciao_test_data.js index 2ccee53c..da1e9821 100644 --- a/test/ciao_test_data.js +++ b/test/ciao_test_data.js @@ -15,6 +15,7 @@ var client = require('elasticsearch').Client(), async = require('async'), actions = []; +var config = require('pelias-config').generate().api; // add one record per 'type' in order to cause the _default_ mapping // to be copied when the new type is created. @@ -28,7 +29,7 @@ types.forEach( function( type, i1 ){ layers.forEach( function( layer, i3 ){ actions.push( function( done ){ client.index({ - index: 'pelias', + index: config.indexName, type: type, id: [i1,i2,i3].join(':'), body: { @@ -49,7 +50,7 @@ types.forEach( function( type, i1 ){ // call refresh so the index merges the changes actions.push( function( done ){ - client.indices.refresh( { index: 'pelias' }, done); + client.indices.refresh( { index: config.indexName }, done); }); // perform all actions in series diff --git a/test/unit/controller/place.js b/test/unit/controller/place.js index 977be59e..0376eec6 100644 --- a/test/unit/controller/place.js +++ b/test/unit/controller/place.js @@ -11,6 +11,11 @@ module.exports.tests.interface = function(test, common) { }); }; +// reminder: this is only the api subsection of the full config +var fakeDefaultConfig = { + indexName: 'pelias' +}; + // functionally test controller (backend success) module.exports.tests.functional_success = function(test, common) { @@ -43,7 +48,37 @@ module.exports.tests.functional_success = function(test, common) { var backend = mockBackend( 'client/mget/ok/1', function( cmd ){ t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: [ 'a' ] } ] } }, 'correct backend command'); }); - var controller = setup( backend ); + var controller = setup( fakeDefaultConfig, backend ); + var res = { + status: function( code ){ + t.equal(code, 200, 'status set'); + return res; + }, + json: function( json ){ + t.equal(typeof json, 'object', 'returns json'); + t.equal(typeof json.date, 'number', 'date set'); + t.equal(json.type, 'FeatureCollection', 'valid geojson'); + t.true(Array.isArray(json.features), 'features is array'); + t.deepEqual(json.features, expected, 'values correctly mapped'); + } + }; + var req = { clean: { ids: [ {'id' : 123, layers: [ 'a' ] } ] }, errors: [], warnings: [] }; + var next = function next() { + t.equal(req.errors.length, 0, 'next was called without error'); + t.end(); + }; + controller(req, res, next ); + }); + + test('functional success with custom index name', function(t) { + var fakeCustomizedConfig = { + indexName: 'alternateindexname' + }; + + var backend = mockBackend( 'client/mget/ok/1', function( cmd ){ + t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'alternateindexname', _type: [ 'a' ] } ] } }, 'correct backend command'); + }); + var controller = setup( fakeCustomizedConfig, backend ); var res = { status: function( code ){ t.equal(code, 200, 'status set'); @@ -72,7 +107,7 @@ module.exports.tests.functional_failure = function(test, common) { var backend = mockBackend( 'client/mget/fail/1', function( cmd ){ t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: ['b'] } ] } }, 'correct backend command'); }); - var controller = setup( backend ); + var controller = setup( fakeDefaultConfig, backend ); var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] }; var next = function( message ){ t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler'); diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js index 09127167..3e7494ac 100644 --- a/test/unit/controller/search.js +++ b/test/unit/controller/search.js @@ -1,4 +1,3 @@ - var setup = require('../../../controller/search'), mockBackend = require('../mock/backend'), mockQuery = require('../mock/query'); @@ -13,6 +12,11 @@ module.exports.tests.interface = function(test, common) { }); }; +// reminder: this is only the api subsection of the full config +var fakeDefaultConfig = { + indexName: 'pelias' +}; + // functionally test controller (backend success) module.exports.tests.functional_success = function(test, common) { @@ -82,7 +86,7 @@ module.exports.tests.functional_success = function(test, common) { searchType: 'dfs_query_then_fetch' }, 'correct backend command'); }); - var controller = setup(backend, mockQuery()); + var controller = setup(fakeDefaultConfig, backend, mockQuery()); var res = { status: function (code) { t.equal(code, 200, 'status set'); @@ -105,6 +109,33 @@ module.exports.tests.functional_success = function(test, common) { }; controller(req, res, next); }); + + test('functional success with alternate index name', function(t) { + var fakeCustomizedConfig = { + indexName: 'alternateindexname' + }; + + var backend = mockBackend('client/search/ok/1', function (cmd) { + t.deepEqual(cmd, { + body: {a: 'b'}, + index: 'alternateindexname', + searchType: 'dfs_query_then_fetch' + }, 'correct backend command'); + }); + var controller = setup(fakeCustomizedConfig, backend, mockQuery()); + var res = { + status: function (code) { + t.equal(code, 200, 'status set'); + return res; + } + }; + var req = { clean: { a: 'b' }, errors: [], warnings: [] }; + var next = function next() { + t.equal(req.errors.length, 0, 'next was called without error'); + t.end(); + }; + controller(req, res, next); + }); }; // functionally test controller (backend failure) @@ -113,7 +144,7 @@ module.exports.tests.functional_failure = function(test, common) { var backend = mockBackend( 'client/search/fail/1', function( cmd ){ t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command'); }); - var controller = setup( backend, mockQuery() ); + var controller = setup( fakeDefaultConfig, backend, mockQuery() ); var req = { clean: { a: 'b' }, errors: [], warnings: [] }; var next = function(){ t.equal(req.errors[0],'a backend error occurred'); @@ -128,7 +159,7 @@ module.exports.tests.timeout = function(test, common) { var backend = mockBackend( 'client/search/timeout/1', function( cmd ){ t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command'); }); - var controller = setup( backend, mockQuery() ); + var controller = setup( fakeDefaultConfig, backend, mockQuery() ); var req = { clean: { a: 'b' }, errors: [], warnings: [] }; var next = function(){ t.equal(req.errors[0],'Request Timeout after 5000ms'); diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js index 6af4e840..94f867b2 100644 --- a/test/unit/fixture/search_boundary_country.js +++ b/test/unit/fixture/search_boundary_country.js @@ -81,7 +81,16 @@ module.exports = { 'weight': 2 }] } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'test' + ] + } + } + ] } }, 'sort': [ '_score' ], diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address.js index 6af4b35b..d6a1478e 100644 --- a/test/unit/fixture/search_full_address.js +++ b/test/unit/fixture/search_full_address.js @@ -128,7 +128,23 @@ module.exports = { 'query': 'new york', 'analyzer': 'peliasAdmin' } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'address', + 'venue', + 'country', + 'region', + 'county', + 'neighbourhood', + 'locality', + 'localadmin' + ] + } + } + ] } }, 'size': 10, diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js index 0a8f5ddc..b8dbf3a1 100644 --- a/test/unit/fixture/search_linguistic_bbox.js +++ b/test/unit/fixture/search_linguistic_bbox.js @@ -74,14 +74,21 @@ module.exports = { }], 'filter': [{ 'geo_bounding_box': { + 'type': 'indexed', 'center_point': { 'top': 11.51, 'right': -61.84, 'bottom': 47.47, 'left': -103.16 - }, - 'type': 'indexed' - } + } + } + }, + { + 'terms': { + 'layer': [ + 'test' + ] + } }] } }, diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js index e3c027c1..38273273 100644 --- a/test/unit/fixture/search_linguistic_focus.js +++ b/test/unit/fixture/search_linguistic_focus.js @@ -101,7 +101,16 @@ module.exports = { 'weight': 2 }] } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'test' + ] + } + } + ] } }, 'sort': [ '_score' ], diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js index a185ea73..ebc5f701 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox.js +++ b/test/unit/fixture/search_linguistic_focus_bbox.js @@ -104,13 +104,20 @@ module.exports = { }], 'filter': [{ 'geo_bounding_box': { + 'type': 'indexed', 'center_point': { 'top': 11.51, 'right': -61.84, 'bottom': 47.47, 'left': -103.16 - }, - 'type': 'indexed' + } + } + }, + { + 'terms': { + 'layer': [ + 'test' + ] } }] } diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js index 152a0c42..8f6fe381 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island.js +++ b/test/unit/fixture/search_linguistic_focus_null_island.js @@ -101,6 +101,13 @@ module.exports = { 'weight': 2 }] } + }], + 'filter':[{ + 'terms': { + 'layer': [ + 'test' + ] + } }] } }, diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js index 2e820c80..490eb0c9 100644 --- a/test/unit/fixture/search_linguistic_only.js +++ b/test/unit/fixture/search_linguistic_only.js @@ -71,7 +71,16 @@ module.exports = { 'weight': 2 }] } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'test' + ] + } + } + ] } }, 'sort': [ '_score' ], diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js index 4f225a62..ca6414a7 100644 --- a/test/unit/fixture/search_linguistic_viewport.js +++ b/test/unit/fixture/search_linguistic_viewport.js @@ -113,6 +113,15 @@ module.exports = { 'boost_mode': 'replace' } } + ], + 'filter': [ + { + 'terms': { + 'layer': [ + 'test' + ] + } + } ] } }, diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address.js index ec8ff4ea..10a2bb74 100644 --- a/test/unit/fixture/search_partial_address.js +++ b/test/unit/fixture/search_partial_address.js @@ -96,7 +96,23 @@ module.exports = { 'query': 'new york', 'analyzer': 'peliasAdmin' } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'address', + 'venue', + 'country', + 'region', + 'county', + 'neighbourhood', + 'locality', + 'localadmin' + ] + } + } + ] } }, 'size': 10, diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address.js index cf65b199..400f561b 100644 --- a/test/unit/fixture/search_regions_address.js +++ b/test/unit/fixture/search_regions_address.js @@ -112,7 +112,23 @@ module.exports = { 'query': 'manhattan', 'analyzer': 'peliasAdmin' } - }] + }], + 'filter': [ + { + 'terms': { + 'layer': [ + 'address', + 'venue', + 'country', + 'region', + 'county', + 'neighbourhood', + 'locality', + 'localadmin' + ] + } + } + ] } }, 'size': 10, diff --git a/test/unit/helper/labelGenerator_USA.js b/test/unit/helper/labelGenerator_USA.js index bdcc5f2f..3097bcca 100644 --- a/test/unit/helper/labelGenerator_USA.js +++ b/test/unit/helper/labelGenerator_USA.js @@ -51,6 +51,25 @@ module.exports.tests.united_states = function(test, common) { t.end(); }); +test('county value should be used when there is no localadmin', function(t) { + var doc = { + 'name': 'venue name', + 'layer': 'venue', + 'housenumber': 'house number', + 'street': 'street name', + 'neighbourhood': 'neighbourhood name', + 'county': 'county name', + 'macrocounty': 'macrocounty name', + 'region_a': 'region abbr', + 'region': 'region name', + 'macroregion': 'macroregion name', + 'country_a': 'USA', + 'country': 'United States' + }; + t.equal(generator(doc),'venue name, county name, region abbr, USA'); + t.end(); + }); + test('street', function(t) { var doc = { 'name': 'house number street name', diff --git a/test/unit/sanitiser/_ids.js b/test/unit/sanitiser/_ids.js index 64e749b0..b6b30a5a 100644 --- a/test/unit/sanitiser/_ids.js +++ b/test/unit/sanitiser/_ids.js @@ -69,7 +69,8 @@ module.exports.tests.invalid_ids = function(test, common) { test('invalid id: source name invalid', function(t) { var raw = { ids: 'invalidsource:venue:23' }; var clean = {}; - var expected_error = 'invalidsource is invalid. It must be one of these values - [' + type_mapping.sources.join(', ') + ']'; + var expected_error = 'invalidsource is invalid. It must be one of these values - [' + + Object.keys(type_mapping.source_mapping).join(', ') + ']'; var messages = sanitize(raw, clean); @@ -107,6 +108,22 @@ module.exports.tests.valid_ids = function(test, common) { t.end(); }); +test('ids: valid short input (openaddresses)', function(t) { + var raw = { ids: 'oa:address:20' }; + var clean = {}; + + var messages = sanitize( raw, clean ); + + var expected_ids = [{ + source: 'openaddresses', + layer: 'address', + id: '20', + }]; + t.deepEqual( messages.errors, [], ' no errors'); + t.deepEqual( clean.ids, expected_ids, 'single type value returned'); + t.end(); + }); + test('ids: valid input (osm)', function(t) { var raw = { ids: 'openstreetmap:venue:node:500' }; var clean = {}; @@ -122,6 +139,22 @@ module.exports.tests.valid_ids = function(test, common) { t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field'); t.end(); }); + + test('ids: valid short input (osm)', function(t) { + var raw = { ids: 'osm:venue:node:500' }; + var clean = {}; + var expected_ids = [{ + source: 'openstreetmap', + layer: 'venue', + id: 'node:500', + }]; + + var messages = sanitize( raw, clean ); + + t.deepEqual( messages.errors, [], ' no errors'); + t.deepEqual( clean.ids, expected_ids, 'osm has node: or way: in id field'); + t.end(); + }); }; module.exports.tests.multiple_ids = function(test, common) {