From e9eed8d4ea85fd4404d20c81be339f41b0a020d0 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 14 Nov 2017 11:27:29 -0500 Subject: [PATCH 01/61] disable package-lock --- .npmrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 .npmrc diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..43c97e71 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false From f56deffb8f9e8cf86fe155bcb5285489030d234d Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 14 Nov 2017 11:28:13 -0500 Subject: [PATCH 02/61] allow libpostal configuration in schema w/other services --- schema.js | 5 +++++ test/unit/schema.js | 15 +++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/schema.js b/schema.js index 3203bb7d..440da818 100644 --- a/schema.js +++ b/schema.js @@ -41,6 +41,11 @@ module.exports = Joi.object().keys({ url: Joi.string().uri({ scheme: /https?/ }), timeout: Joi.number().integer().optional().default(250).min(0), retries: Joi.number().integer().optional().default(3).min(0), + }).unknown(false).requiredKeys('url'), + libpostal: Joi.object().keys({ + url: Joi.string().uri({ scheme: /https?/ }), + timeout: Joi.number().integer().optional().default(250).min(0), + retries: Joi.number().integer().optional().default(3).min(0), }).unknown(false).requiredKeys('url') }).unknown(false).default({}), // default api.services to an empty object defaultParameters: Joi.object().keys({ diff --git a/test/unit/schema.js b/test/unit/schema.js index 7c62f109..1278ee59 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -28,6 +28,9 @@ module.exports.tests.completely_valid = (test, common) => { }, interpolation: { url: 'http://localhost' + }, + libpostal: { + url: 'http://localhost' } }, defaultParameters: { @@ -419,7 +422,7 @@ module.exports.tests.api_validation = (test, common) => { }); // api.pipService has been moved to api.services.pip.url - test('any api.pipService value should be allowed', (t) => { + test('any api.pipService value should fail', (t) => { [null, 17, {}, [], true, 'http://localhost'].forEach((value) => { var config = { api: { @@ -543,7 +546,7 @@ module.exports.tests.api_services_validation = (test, common) => { module.exports.tests.service_validation = (test, common) => { // these tests apply for all the individual service definitions - const services = ['pip', 'placeholder', 'interpolation']; + const services = ['pip', 'placeholder', 'interpolation', 'libpostal']; test('timeout and retries not specified should default to 250 and 3', (t) => { services.forEach(service => { @@ -572,7 +575,7 @@ module.exports.tests.service_validation = (test, common) => { }); - test('when api.services.service is defined, url is required', (t) => { + test('when api.services. is defined, url is required', (t) => { services.forEach(service => { const config = { api: { @@ -596,7 +599,7 @@ module.exports.tests.service_validation = (test, common) => { }); - test('non-string api.services.pip.url should throw error', (t) => { + test('non-string api.services..url should throw error', (t) => { services.forEach(service => { [null, 17, {}, [], true].forEach(value => { const config = { @@ -626,7 +629,7 @@ module.exports.tests.service_validation = (test, common) => { }); - test('non-http/https api.services.pip.url should throw error', (t) => { + test('non-http/https api.services..url should throw error', (t) => { services.forEach(service => { ['ftp', 'git', 'unknown'].forEach((scheme) => { const config = { @@ -656,7 +659,7 @@ module.exports.tests.service_validation = (test, common) => { }); - test('non-url children of api.services.pip should be disallowed', (t) => { + test('non-url/timeout/retries children of api.services. should be disallowed', (t) => { services.forEach(service => { const config = { api: { From b5e48afb352b651f2c80783aceef40931302c89b Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 14 Nov 2017 11:29:54 -0500 Subject: [PATCH 03/61] remove text-analyzer address operation --- sanitizer/_synthesize_analysis.js | 44 +----- test/unit/sanitizer/_synthesize_analysis.js | 153 +------------------- 2 files changed, 4 insertions(+), 193 deletions(-) diff --git a/sanitizer/_synthesize_analysis.js b/sanitizer/_synthesize_analysis.js index 3822d9d4..da67c752 100644 --- a/sanitizer/_synthesize_analysis.js +++ b/sanitizer/_synthesize_analysis.js @@ -1,5 +1,4 @@ const _ = require('lodash'); -const text_analyzer = require('pelias-text-analyzer'); const fields = { 'venue': 'query', @@ -17,20 +16,6 @@ function normalizeWhitespaceToSingleSpace(val) { return _.replace(_.trim(val), /\s+/g, ' '); } -function isPostalCodeOnly(parsed_text) { - return Object.keys(parsed_text).length === 1 && - parsed_text.hasOwnProperty('postalcode'); -} - -// figure out which field contains the probable house number, prefer number -// libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street -// so because we're treating the entire field as a street address, it's safe -// to assume that an identified postcode is actually a house number. -function getHouseNumberField(analyzed_address) { - // return the first field available in the libpostal response, undefined if none - return _.find(['number', 'postalcode'], _.partial(_.has, analyzed_address)); -} - function _sanitize( raw, clean ){ // error & warning messages @@ -51,35 +36,8 @@ function _sanitize( raw, clean ){ `at least one of the following fields is required: ${Object.keys(fields).join(', ')}`); } - if (clean.parsed_text.hasOwnProperty('address')) { - const analyzed_address = text_analyzer.parse(clean.parsed_text.address); - - const house_number_field = getHouseNumberField(analyzed_address); - - // if we're fairly certain that libpostal identified a house number - // (from either the house_number or postcode field), place it into the - // number field and remove the first instance of that value from address - // and assign to street - // eg - '1090 N Charlotte St' becomes number=1090 and street=N Charlotte St - if (house_number_field) { - clean.parsed_text.number = analyzed_address[house_number_field]; - - // remove the first instance of the number and trim whitespace - clean.parsed_text.street = _.trim(_.replace(clean.parsed_text.address, clean.parsed_text.number, '')); - - } else { - // otherwise no house number was identifiable, so treat the entire input - // as a street - clean.parsed_text.street = clean.parsed_text.address; - - } - - // the address field no longer means anything since it's been parsed, so remove it - delete clean.parsed_text.address; - - } - return messages; + } function _expected() { diff --git a/test/unit/sanitizer/_synthesize_analysis.js b/test/unit/sanitizer/_synthesize_analysis.js index 64243dc9..19ff57bd 100644 --- a/test/unit/sanitizer/_synthesize_analysis.js +++ b/test/unit/sanitizer/_synthesize_analysis.js @@ -1,18 +1,14 @@ const _ = require('lodash'); const proxyquire = require('proxyquire').noCallThru(); +const sanitizer = require('../../../sanitizer/_synthesize_analysis'); module.exports.tests = {}; module.exports.tests.text_parser = function(test, common) { test('all variables should be parsed', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.fail('parse should not have been called'); - } - }}); - const raw = { venue: ' \t venue \t value \t ', + address: ' \t address \t value \t ', neighbourhood: ' \t neighbourhood \t value \t ', borough: ' \t borough \t value \t ', locality: ' \t locality \t value \t ', @@ -27,6 +23,7 @@ module.exports.tests.text_parser = function(test, common) { const expected_clean = { parsed_text: { query: 'venue value', + address: 'address value', neighbourhood: 'neighbourhood value', borough: 'borough value', city: 'locality value', @@ -47,12 +44,6 @@ module.exports.tests.text_parser = function(test, common) { }); test('non-string and blank string values should be treated as not supplied', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.fail('parse should not have been called'); - } - }}); - // helper to return a random value that's considered invalid function getInvalidValue() { return _.sample([{}, [], false, '', ' \t ', 17, undefined]); @@ -87,12 +78,6 @@ module.exports.tests.text_parser = function(test, common) { }); test('no supplied fields should return error', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.fail('parse should not have been called'); - } - }}); - const raw = {}; const clean = {}; @@ -110,12 +95,6 @@ module.exports.tests.text_parser = function(test, common) { }); test('postalcode-only parsed_text should return error', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.fail('parse should not have been called'); - } - }}); - const raw = { postalcode: 'postalcode value' }; @@ -137,132 +116,6 @@ module.exports.tests.text_parser = function(test, common) { }); - test('text_analyzer identifying house number should extract it and street', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.equals(query, 'Number Value Street Value Number Value'); - - return { - number: 'Number Value' - }; - } - }}); - - const raw = { - address: 'Number Value Street Value Number Value' - }; - - const clean = {}; - - const expected_clean = { - parsed_text: { - number: 'Number Value', - street: 'Street Value Number Value' - } - }; - - const messages = sanitizer().sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('text_analyzer identifying postalcode but not house number should assign to number and remove from address', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.equals(query, 'Number Value Street Value Number Value'); - - return { - postalcode: 'Number Value' - }; - } - }}); - - const raw = { - address: 'Number Value Street Value Number Value' - }; - - const clean = {}; - - const expected_clean = { - parsed_text: { - number: 'Number Value', - street: 'Street Value Number Value' - } - }; - - const messages = sanitizer().sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('text_analyzer not revealing possible number should move address to street', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.equals(query, 'Street Value'); - - return {}; - } - }}); - - const raw = { - address: 'Street Value' - }; - - const clean = {}; - - const expected_clean = { - parsed_text: { - street: 'Street Value' - } - }; - - const messages = sanitizer().sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('text_analyzer returning undefined on address resolution should treat as if no house number field was found', t => { - var sanitizer = proxyquire('../../../sanitizer/_synthesize_analysis', { - 'pelias-text-analyzer': { parse: function(query) { - t.equals(query, 'Street Value'); - - return undefined; - } - }}); - - const raw = { - address: 'Street Value' - }; - - const clean = {}; - - const expected_clean = { - parsed_text: { - street: 'Street Value' - } - }; - - const messages = sanitizer().sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - test('return an array of expected parameters in object form for validation', function (t) { const sanitizer = require('../../../sanitizer/_synthesize_analysis'); const expected = [ From e0c25d0f573e34203a8d503c3e31f8396a56f55a Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 14 Nov 2017 11:44:20 -0500 Subject: [PATCH 04/61] convert libpostal calls to a microservice --- controller/libpostal.js | 105 ++++- controller/structured_libpostal.js | 75 +++ package.json | 1 - routes/v1.js | 23 +- service/configurations/Libpostal.js | 33 ++ test/unit/controller/libpostal.js | 343 +++++++------- test/unit/controller/structured_libpostal.js | 440 ++++++++++++++++++ test/unit/run.js | 2 + test/unit/service/configurations/Libpostal.js | 99 ++++ 9 files changed, 945 insertions(+), 176 deletions(-) create mode 100644 controller/structured_libpostal.js create mode 100644 service/configurations/Libpostal.js create mode 100644 test/unit/controller/structured_libpostal.js create mode 100644 test/unit/service/configurations/Libpostal.js diff --git a/controller/libpostal.js b/controller/libpostal.js index 23c82f85..7ecbe3b6 100644 --- a/controller/libpostal.js +++ b/controller/libpostal.js @@ -1,31 +1,108 @@ -const text_analyzer = require('pelias-text-analyzer'); const _ = require('lodash'); const iso3166 = require('iso3166-1'); const Debug = require('../helper/debug'); const debugLog = new Debug('controller:libpostal'); +const logger = require('pelias-logger').get('api'); -function setup(should_execute) { +// mapping object from libpostal fields to pelias fields +var field_mapping = { + island: 'island', + category: 'category', + house: 'query', + house_number: 'number', + road: 'street', + suburb: 'neighbourhood', + city_district: 'borough', + city: 'city', + state_district: 'county', + state: 'state', + postcode: 'postalcode', + country: 'country' +}; + +// This controller calls the hosted libpostal service and converts the response +// to a generic format for later use. The hosted service returns an array like: +// +// ``` +// [ +// { +// label: 'house_number', +// value: '30' +// }, +// { +// label: 'road', +// value: 'west 26th street' +// }, +// { +// label: 'city', +// value: 'new york' +// }, +// { +// label: 'state', +// value: 'ny' +// } +//] +// ``` +// +// where `label` can be any of (currently): +// - house (generally interpreted as unknown, treated by pelias like a query term) +// - category (like "restaurants") +// - house_number +// - road +// - unit (apt or suite #) +// - suburb (like a neighbourhood) +// - city +// - city_district (like an NYC borough) +// - state_district (like a county) +// - state +// - postcode +// - country +// +// The Pelias query module is not concerned with unit. +// +function setup(libpostalService, should_execute) { function controller( req, res, next ){ // bail early if req/res don't pass conditions for execution if (!should_execute(req, res)) { return next(); } + const initialTime = debugLog.beginTimer(req); - // parse text with query parser - const parsed_text = text_analyzer.parse(req.clean.text); - if (parsed_text !== undefined) { - // if a known ISO2 country was parsed, convert it to ISO3 - if (_.has(parsed_text, 'country') && iso3166.is2(_.toUpper(parsed_text.country))) { - parsed_text.country = iso3166.to3(_.toUpper(parsed_text.country)); + libpostalService(req, (err, response) => { + if (err) { + // push err.message or err onto req.errors + req.errors.push( _.get(err, 'message', err) ); + + } else if (_.some(_.countBy(response, o => o.label), count => count > 1)) { + logger.warn(`discarding libpostal parse of '${req.clean.text}' due to duplicate field assignments`); + return next(); + + } else if (_.isEmpty(response)) { + return next(); + + } else { + req.clean.parser = 'libpostal'; + req.clean.parsed_text = response.reduce(function(o, f) { + if (field_mapping.hasOwnProperty(f.label)) { + o[field_mapping[f.label]] = f.value; + } + + return o; + }, {}); + + if (_.has(req.clean.parsed_text, 'country') && iso3166.is2(_.toUpper(req.clean.parsed_text.country))) { + req.clean.parsed_text.country = iso3166.to3(_.toUpper(req.clean.parsed_text.country)); + } + + debugLog.push(req, {parsed_text: req.clean.parsed_text}); + } - req.clean.parser = 'libpostal'; - req.clean.parsed_text = parsed_text; - debugLog.push(req, {parsed_text: req.clean.parsed_text}); - } - debugLog.stopTimer(req, initialTime); - return next(); + debugLog.stopTimer(req, initialTime); + return next(); + + }); } diff --git a/controller/structured_libpostal.js b/controller/structured_libpostal.js new file mode 100644 index 00000000..626bf945 --- /dev/null +++ b/controller/structured_libpostal.js @@ -0,0 +1,75 @@ +const _ = require('lodash'); +const Debug = require('../helper/debug'); +const debugLog = new Debug('controller:libpostal'); +const logger = require('pelias-logger').get('api'); + +// if there's a house_number in the libpostal response, return it +// otherwise return the postcode field (which may be undefined) +function findHouseNumberField(response) { + const house_number_field = response.find(f => f.label === 'house_number'); + + if (house_number_field) { + return house_number_field; + } + + return response.find(f => f.label === 'postcode'); + +} + +function setup(libpostalService, should_execute) { + function controller( req, res, next ){ + // bail early if req/res don't pass conditions for execution + if (!should_execute(req, res)) { + return next(); + } + + const initialTime = debugLog.beginTimer(req); + + libpostalService(req, (err, response) => { + if (err) { + // push err.message or err onto req.errors + req.errors.push( _.get(err, 'message', err) ); + + } else { + // figure out which field contains the probable house number, prefer house_number + // libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street + // so because we're treating the entire field as a street address, it's safe + // to assume that an identified postcode is actually a house number. + const house_number_field = findHouseNumberField(response); + + // if we're fairly certain that libpostal identified a house number + // (from either the house_number or postcode field), place it into the + // number field and remove the first instance of that value from address + // and assign to street + // eg - '1090 N Charlotte St' becomes number=1090 and street=N Charlotte St + if (house_number_field) { + req.clean.parsed_text.number = house_number_field.value; + + // remove the first instance of the number and trim whitespace + req.clean.parsed_text.street = _.trim(_.replace(req.clean.parsed_text.address, req.clean.parsed_text.number, '')); + + } else { + // otherwise no house number was identifiable, so treat the entire input + // as a street + req.clean.parsed_text.street = req.clean.parsed_text.address; + + } + + // the address field no longer means anything since it's been parsed, so remove it + delete req.clean.parsed_text.address; + + debugLog.push(req, {parsed_text: response}); + + } + + debugLog.stopTimer(req, initialTime); + return next(); + + }); + + } + + return controller; +} + +module.exports = setup; diff --git a/package.json b/package.json index 80243ad1..d32e37cc 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,6 @@ "pelias-model": "5.2.0", "pelias-query": "9.1.1", "pelias-sorting": "1.1.0", - "pelias-text-analyzer": "1.10.2", "predicates": "^2.0.0", "retry": "^0.10.1", "stats-lite": "^2.0.4", diff --git a/routes/v1.js b/routes/v1.js index d300d20e..fb3539a8 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -29,6 +29,7 @@ var controllers = { coarse_reverse: require('../controller/coarse_reverse'), mdToHTML: require('../controller/markdownToHtml'), libpostal: require('../controller/libpostal'), + structured_libpostal: require('../controller/structured_libpostal'), place: require('../controller/place'), placeholder: require('../controller/placeholder'), search: require('../controller/search'), @@ -96,6 +97,7 @@ const PlaceHolder = require('../service/configurations/PlaceHolder'); const PointInPolygon = require('../service/configurations/PointInPolygon'); const Language = require('../service/configurations/Language'); const Interpolation = require('../service/configurations/Interpolation'); +const Libpostal = require('../service/configurations/Libpostal'); /** * Append routes to app @@ -122,6 +124,18 @@ function addRoutes(app, peliasConfig) { const interpolationService = serviceWrapper(interpolationConfiguration); const isInterpolationEnabled = _.constant(interpolationConfiguration.isEnabled()); + // standard libpostal should use req.clean.text for the `address` parameter + const libpostalConfiguration = new Libpostal( + _.defaultTo(peliasConfig.api.services.libpostal, {}), + _.property('clean.text')); + const libpostalService = serviceWrapper(libpostalConfiguration); + + // structured libpostal should use req.clean.parsed_text.address for the `address` parameter + const structuredLibpostalConfiguration = new Libpostal( + _.defaultTo(peliasConfig.api.services.libpostal, {}), + _.property('clean.parsed_text.address')); + const structuredLibpostalService = serviceWrapper(structuredLibpostalConfiguration); + // fallback to coarse reverse when regular reverse didn't return anything const coarseReverseShouldExecute = all( isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData) @@ -132,6 +146,12 @@ function addRoutes(app, peliasConfig) { not(isRequestSourcesOnlyWhosOnFirst) ); + // for libpostal to execute for structured requests, req.clean.parsed_text.address must exist + const structuredLibpostalShouldExecute = all( + not(hasRequestErrors), + hasParsedTextProperties.all('address') + ); + // execute placeholder if libpostal only parsed as admin-only and needs to // be geodisambiguated const placeholderGeodisambiguationShouldExecute = all( @@ -256,7 +276,7 @@ function addRoutes(app, peliasConfig) { sanitizers.search.middleware(peliasConfig.api), middleware.requestLanguage, middleware.calcSize(), - controllers.libpostal(libpostalShouldExecute), + controllers.libpostal(libpostalService, libpostalShouldExecute), controllers.placeholder(placeholderService, geometricFiltersApply, placeholderGeodisambiguationShouldExecute), controllers.placeholder(placeholderService, geometricFiltersDontApply, placeholderIdsLookupShouldExecute), controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute), @@ -286,6 +306,7 @@ function addRoutes(app, peliasConfig) { sanitizers.structured_geocoding.middleware(peliasConfig.api), middleware.requestLanguage, middleware.calcSize(), + controllers.structured_libpostal(structuredLibpostalService, structuredLibpostalShouldExecute), controllers.search(peliasConfig.api, esclient, queries.structured_geocoding, not(hasResponseDataOrRequestErrors)), postProc.trimByGranularityStructured(), postProc.distances('focus.point.'), diff --git a/service/configurations/Libpostal.js b/service/configurations/Libpostal.js new file mode 100644 index 00000000..9cf1de2b --- /dev/null +++ b/service/configurations/Libpostal.js @@ -0,0 +1,33 @@ +'use strict'; + +const url = require('url'); + +const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration; + +class Libpostal extends ServiceConfiguration { + constructor(o, propertyExtractor) { + super('libpostal', o); + + // save off the propertyExtractor function + // this is used to extract a single property from req. eg: + // * _.property('clean.text') + // * _.property('clean.parsed_text.address') + // will return those properties from req + this.propertyExtractor = propertyExtractor; + + } + + getParameters(req) { + return { + address: this.propertyExtractor(req) + }; + + } + + getUrl(req) { + return url.resolve(this.baseUrl, 'parse'); + } + +} + +module.exports = Libpostal; diff --git a/test/unit/controller/libpostal.js b/test/unit/controller/libpostal.js index f117007d..cf62fff5 100644 --- a/test/unit/controller/libpostal.js +++ b/test/unit/controller/libpostal.js @@ -1,281 +1,304 @@ 'use strict'; const proxyquire = require('proxyquire').noCallThru(); +const libpostal = require('../../../controller/libpostal'); const _ = require('lodash'); +const mock_logger = require('pelias-mock-logger'); module.exports.tests = {}; module.exports.tests.interface = (test, common) => { - test('valid interface', t => { - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => undefined - } - }); - - t.equal(typeof controller, 'function', 'libpostal is a function'); - t.equal(typeof controller(), 'function', 'libpostal returns a controller'); + test('valid interface', (t) => { + t.equal(typeof libpostal, 'function', 'libpostal is a function'); + t.equal(typeof libpostal(), 'function', 'libpostal returns a controller'); t.end(); - }); - }; -module.exports.tests.should_execute = (test, common) => { - test('should_execute returning false should not call text-analyzer', t => { - const should_execute = (req, res) => { +module.exports.tests.early_exit_conditions = (test, common) => { + test('should_execute returning false should not call service', t => { + const service = () => { + t.fail('service should not have been called'); + }; + + const should_execute = (req) => { // req and res should be passed to should_execute t.deepEquals(req, { clean: { text: 'original query' } }); - t.deepEquals(res, { b: 2 }); + return false; }; - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => { - t.fail('parse should not have been called'); - } - } - })(should_execute); + const controller = libpostal(service, should_execute); const req = { clean: { text: 'original query' } }; - const res = { b: 2 }; - controller(req, res, () => { + controller(req, undefined, () => { t.deepEquals(req, { clean: { text: 'original query' } }, 'req should not have been modified'); - t.deepEquals(res, { b: 2 }); + t.end(); }); }); - test('should_execute returning false should not call text-analyzer', t => { - t.plan(5); +}; - const should_execute = (req, res) => { - // req and res should be passed to should_execute - t.deepEquals(req, { - clean: { - text: 'original query' - } - }); - t.deepEquals(res, { b: 2 }); - return true; +module.exports.tests.error_conditions = (test, common) => { + test('service returning error should append and not modify req.clean', t => { + const service = (req, callback) => { + callback('libpostal service error', []); }; - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: (query) => { - t.equals(query, 'original query'); - return undefined; - } - } - })(should_execute); + const controller = libpostal(service, () => true); const req = { clean: { text: 'original query' - } + }, + errors: [] }; - const res = { b: 2 }; - controller(req, res, () => { + controller(req, undefined, () => { t.deepEquals(req, { clean: { text: 'original query' - } + }, + errors: ['libpostal service error'] }, 'req should not have been modified'); - t.deepEquals(res, { b: 2 }); + t.end(); + }); }); }; -module.exports.tests.parse_is_called = (test, common) => { - test('parse returning undefined should not overwrite clean.parsed_text', t => { - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => undefined - } - })(() => true); - - const req = { - clean: { - parsed_text: 'original parsed_text' - } - }; - const res = 'this is the response'; +module.exports.tests.failure_conditions = (test, common) => { + test('service returning 2 or more of a label should return undefined and log message', t => { + const logger = mock_logger(); - controller(req, res, () => { - t.deepEquals(req, { - clean: { - parsed_text: 'original parsed_text' + const service = (req, callback) => { + const response = [ + { + label: 'road', + value: 'road value 1' + }, + { + label: 'city', + value: 'city value' + }, + { + label: 'road', + value: 'road value 2' } - }); - t.deepEquals(res, 'this is the response'); - t.end(); - }); + ]; - }); + callback(null, response); + }; - test('parse returning something should overwrite clean.parsed_text', t => { const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => 'replacement parsed_text' - } - })(() => true); + 'pelias-logger': logger + })(service, () => true); const req = { clean: { - parsed_text: 'original parsed_text' - } + text: 'query value' + }, + errors: [] }; - const res = 'this is the response'; - controller(req, res, () => { + controller(req, undefined, () => { + t.ok(logger.isWarnMessage('discarding libpostal parse of \'query value\' due to duplicate field assignments')); + t.deepEquals(req, { clean: { - parser: 'libpostal', - parsed_text: 'replacement parsed_text' - } - }); - t.deepEquals(res, 'this is the response'); + text: 'query value' + }, + errors: [] + }, 'req should not have been modified'); + t.end(); + }); }); -}; + test('service returning empty array should not set parsed_text or parser', t => { + const logger = mock_logger(); + + const service = (req, callback) => { + callback(null, []); + }; -module.exports.tests.iso2_conversion = (test, common) => { - test('no country in parse response should not leave country unset', t => { const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => ({ - locality: 'this is the locality' - }) - }, - 'iso3166-1': { - is2: () => t.fail('should not have been called'), - to3: () => t.fail('should not have been called') - } - })(() => true); + 'pelias-logger': logger + })(service, () => true); const req = { clean: { - parsed_text: 'original parsed_text' - } + text: 'query value' + }, + errors: [] }; - const res = 'this is the response'; - controller(req, res, () => { + controller(req, undefined, () => { t.deepEquals(req, { clean: { - parser: 'libpostal', - parsed_text: { - locality: 'this is the locality' - } - } - }); - t.deepEquals(res, 'this is the response'); + text: 'query value' + }, + errors: [] + }, 'req should not have been modified'); + t.end(); + }); }); - test('unknown country should not be converted', t => { - t.plan(3); +}; - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => ({ - country: 'unknown country code' - }) - }, - 'iso3166-1': { - is2: country => { - t.equals(country, 'UNKNOWN COUNTRY CODE'); - return false; +module.exports.tests.success_conditions = (test, common) => { + test('service returning valid response should convert and append', t => { + const service = (req, callback) => { + const response = [ + { + label: 'island', + value: 'island value' }, - to3: () => t.fail('should not have been called') - } - })(() => true); + { + label: 'category', + value: 'category value' + }, + { + label: 'house', + value: 'house value' + }, + { + label: 'house_number', + value: 'house_number value' + }, + { + label: 'road', + value: 'road value' + }, + { + label: 'suburb', + value: 'suburb value' + }, + { + label: 'city_district', + value: 'city_district value' + }, + { + label: 'city', + value: 'city value' + }, + { + label: 'state_district', + value: 'state_district value' + }, + { + label: 'state', + value: 'state value' + }, + { + label: 'postcode', + value: 'postcode value' + }, + { + label: 'country', + value: 'country value' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); const req = { clean: { - parsed_text: 'original parsed_text' - } + text: 'original query' + }, + errors: [] }; - const res = 'this is the response'; - controller(req, res, () => { + controller(req, undefined, () => { t.deepEquals(req, { clean: { + text: 'original query', parser: 'libpostal', parsed_text: { - country: 'unknown country code' + island: 'island value', + category: 'category value', + query: 'house value', + number: 'house_number value', + street: 'road value', + neighbourhood: 'suburb value', + borough: 'city_district value', + city: 'city value', + county: 'state_district value', + state: 'state value', + postalcode: 'postcode value', + country: 'country value' } - } - }); - t.deepEquals(res, 'this is the response'); + }, + errors: [] + }, 'req should not have been modified'); + t.end(); + }); }); - test('ISO2 country should be converted to ISO3', t => { - t.plan(4); - - const controller = proxyquire('../../../controller/libpostal', { - 'pelias-text-analyzer': { - parse: () => ({ - country: 'ISO2 COUNTRY CODE' - }) - }, - 'iso3166-1': { - is2: country => { - t.equals(country, 'ISO2 COUNTRY CODE'); - return true; - }, - to3: country => { - t.equals(country, 'ISO2 COUNTRY CODE'); - return 'ISO3 COUNTRY CODE'; + test('ISO-2 country should be converted to ISO-3', t => { + const service = (req, callback) => { + const response = [ + { + label: 'country', + value: 'ca' } - } - })(() => true); + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); const req = { clean: { - parsed_text: 'original parsed_text' - } + text: 'original query' + }, + errors: [] }; - const res = 'this is the response'; - controller(req, res, () => { + controller(req, undefined, () => { t.deepEquals(req, { clean: { + text: 'original query', parser: 'libpostal', parsed_text: { - country: 'ISO3 COUNTRY CODE' + country: 'CAN' } - } - }); - t.deepEquals(res, 'this is the response'); + }, + errors: [] + }, 'req should not have been modified'); + t.end(); + }); }); diff --git a/test/unit/controller/structured_libpostal.js b/test/unit/controller/structured_libpostal.js new file mode 100644 index 00000000..eb4aea12 --- /dev/null +++ b/test/unit/controller/structured_libpostal.js @@ -0,0 +1,440 @@ +'use strict'; + +const proxyquire = require('proxyquire').noCallThru(); +const libpostal = require('../../../controller/structured_libpostal'); +const _ = require('lodash'); +const mock_logger = require('pelias-mock-logger'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + t.equal(typeof libpostal, 'function', 'libpostal is a function'); + t.equal(typeof libpostal(), 'function', 'libpostal returns a controller'); + t.end(); + }); +}; + +module.exports.tests.early_exit_conditions = (test, common) => { + test('should_execute returning false should not call service', t => { + const service = () => { + t.fail('service should not have been called'); + }; + + const should_execute = (req) => { + // req and res should be passed to should_execute + t.deepEquals(req, { + clean: { + text: 'original query' + } + }); + + return false; + }; + + const controller = libpostal(service, should_execute); + + const req = { + clean: { + text: 'original query' + } + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + text: 'original query' + } + }, 'req should not have been modified'); + + t.end(); + }); + + }); + +}; + +module.exports.tests.error_conditions = (test, common) => { + test('service returning error should append and not modify req.clean', t => { + const service = (req, callback) => { + callback('libpostal service error', []); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + text: 'original query' + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + text: 'original query' + }, + errors: ['libpostal service error'] + }, 'req should not have been modified'); + + t.end(); + + }); + + }); + +}; + +// module.exports.tests.failure_conditions = (test, common) => { +// test('service returning 2 or more of a label should return undefined and log message', t => { +// const logger = mock_logger(); +// +// const service = (req, callback) => { +// const response = [ +// { +// label: 'road', +// value: 'road value 1' +// }, +// { +// label: 'city', +// value: 'city value' +// }, +// { +// label: 'road', +// value: 'road value 2' +// } +// ]; +// +// callback(null, response); +// }; +// +// const controller = proxyquire('../../../controller/libpostal', { +// 'pelias-logger': logger +// })(service, () => true); +// +// const req = { +// clean: { +// text: 'query value' +// }, +// errors: [] +// }; +// +// controller(req, undefined, () => { +// t.ok(logger.isWarnMessage('discarding libpostal parse of \'query value\' due to duplicate field assignments')); +// +// t.deepEquals(req, { +// clean: { +// text: 'query value' +// }, +// errors: [] +// }, 'req should not have been modified'); +// +// t.end(); +// +// }); +// +// }); +// +// test('service returning empty array should not set parsed_text or parser', t => { +// const logger = mock_logger(); +// +// const service = (req, callback) => { +// callback(null, []); +// }; +// +// const controller = proxyquire('../../../controller/libpostal', { +// 'pelias-logger': logger +// })(service, () => true); +// +// const req = { +// clean: { +// text: 'query value' +// }, +// errors: [] +// }; +// +// controller(req, undefined, () => { +// t.deepEquals(req, { +// clean: { +// text: 'query value' +// }, +// errors: [] +// }, 'req should not have been modified'); +// +// t.end(); +// +// }); +// +// }); +// +// }; +// +module.exports.tests.success_conditions = (test, common) => { + test('service returning house_number should set req.clean.parsed_text.', t => { + const service = (req, callback) => { + const response = [ + { + label: 'house_number', + value: 'house_number value' + }, + { + label: 'postcode', + value: 'postcode value' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + parsed_text: { + address: 'other value house_number value street value' + } + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + parsed_text: { + number: 'house_number value', + street: 'other value street value' + } + }, + errors: [] + }, 'req should not have been modified'); + + t.end(); + + }); + + }); + + test('service returning postcode should set req.clean.parsed_text.', t => { + const service = (req, callback) => { + const response = [ + { + label: 'postcode', + value: 'postcode value' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + parsed_text: { + address: 'other value postcode value street value' + } + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + parsed_text: { + number: 'postcode value', + street: 'other value street value' + } + }, + errors: [] + }, 'req should not have been modified'); + + t.end(); + + }); + + }); + + test('service returning neither house_number nor postcode should not set req.clean.parsed_text.number', t => { + const service = (req, callback) => { + const response = [ + { + label: 'city', + value: 'city value' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + parsed_text: { + address: 'street value' + } + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + parsed_text: { + street: 'street value' + } + }, + errors: [] + }, 'req should not have been modified'); + + t.end(); + + }); + + }); + + // test('service returning valid response should convert and append', t => { + // const service = (req, callback) => { + // const response = [ + // { + // label: 'island', + // value: 'island value' + // }, + // { + // label: 'category', + // value: 'category value' + // }, + // { + // label: 'house', + // value: 'house value' + // }, + // { + // label: 'house_number', + // value: 'house_number value' + // }, + // { + // label: 'road', + // value: 'road value' + // }, + // { + // label: 'suburb', + // value: 'suburb value' + // }, + // { + // label: 'city_district', + // value: 'city_district value' + // }, + // { + // label: 'city', + // value: 'city value' + // }, + // { + // label: 'state_district', + // value: 'state_district value' + // }, + // { + // label: 'state', + // value: 'state value' + // }, + // { + // label: 'postcode', + // value: 'postcode value' + // }, + // { + // label: 'country', + // value: 'country value' + // } + // ]; + // + // callback(null, response); + // }; + // + // const controller = libpostal(service, () => true); + // + // const req = { + // clean: { + // text: 'original query' + // }, + // errors: [] + // }; + // + // controller(req, undefined, () => { + // t.deepEquals(req, { + // clean: { + // text: 'original query', + // parser: 'libpostal', + // parsed_text: { + // island: 'island value', + // category: 'category value', + // query: 'house value', + // number: 'house_number value', + // street: 'road value', + // neighbourhood: 'suburb value', + // borough: 'city_district value', + // city: 'city value', + // county: 'state_district value', + // state: 'state value', + // postalcode: 'postcode value', + // country: 'country value' + // } + // }, + // errors: [] + // }, 'req should not have been modified'); + // + // t.end(); + // + // }); + // + // }); + // + // test('ISO-2 country should be converted to ISO-3', t => { + // const service = (req, callback) => { + // const response = [ + // { + // label: 'country', + // value: 'ca' + // } + // ]; + // + // callback(null, response); + // }; + // + // const controller = libpostal(service, () => true); + // + // const req = { + // clean: { + // text: 'original query' + // }, + // errors: [] + // }; + // + // controller(req, undefined, () => { + // t.deepEquals(req, { + // clean: { + // text: 'original query', + // parser: 'libpostal', + // parsed_text: { + // country: 'CAN' + // } + // }, + // errors: [] + // }, 'req should not have been modified'); + // + // t.end(); + // + // }); + // + // }); + // +}; + +module.exports.all = (tape, common) => { + + function test(name, testFunction) { + return tape(`GET /libpostal ${name}`, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/run.js b/test/unit/run.js index 5631b9f3..ffebf9ea 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -14,6 +14,7 @@ var tests = [ require('./controller/coarse_reverse'), require('./controller/index'), require('./controller/libpostal'), + require('./controller/structured_libpostal'), require('./controller/place'), require('./controller/placeholder'), require('./controller/search'), @@ -100,6 +101,7 @@ var tests = [ require('./sanitizer/wrap'), require('./service/configurations/Interpolation'), require('./service/configurations/Language'), + require('./service/configurations/Libpostal'), require('./service/configurations/PlaceHolder'), require('./service/configurations/PointInPolygon'), require('./service/mget'), diff --git a/test/unit/service/configurations/Libpostal.js b/test/unit/service/configurations/Libpostal.js new file mode 100644 index 00000000..2dcbc78c --- /dev/null +++ b/test/unit/service/configurations/Libpostal.js @@ -0,0 +1,99 @@ +const Libpostal = require('../../../../service/configurations/Libpostal'); + +module.exports.tests = {}; + +module.exports.tests.all = (test, common) => { + test('getName should return \'libpostal\'', (t) => { + const configBlob = { + url: 'http://localhost:1234', + timeout: 17, + retries: 19 + }; + + const libpostal = new Libpostal(configBlob); + + t.equals(libpostal.getName(), 'libpostal'); + t.equals(libpostal.getBaseUrl(), 'http://localhost:1234/'); + t.equals(libpostal.getTimeout(), 17); + t.equals(libpostal.getRetries(), 19); + t.end(); + + }); + + test('getUrl should return value passed to constructor', (t) => { + const configBlob = { + url: 'http://localhost:1234', + timeout: 17, + retries: 19 + }; + + const libpostal = new Libpostal(configBlob); + + t.equals(libpostal.getUrl(), 'http://localhost:1234/parse'); + t.end(); + + }); + + test('getParameters should return object with text and lang from req', (t) => { + const configBlob = { + url: 'http://localhost:1234', + timeout: 17, + retries: 19 + }; + + const propertyExtractor = (req) => { + t.deepEquals(req, { a: 1, b: 2}); + return 'property value'; + }; + + const libpostal = new Libpostal(configBlob, propertyExtractor); + + const req = { + a: 1, + b: 2 + }; + + t.deepEquals(libpostal.getParameters(req), { address: 'property value' }); + t.end(); + + }); + + test('getHeaders should return empty object', (t) => { + const configBlob = { + url: 'base url', + timeout: 17, + retries: 19 + }; + + const libpostal = new Libpostal(configBlob); + + t.deepEquals(libpostal.getHeaders(), {}); + t.end(); + + }); + + test('baseUrl ending in / should not have double /\'s return by getUrl', (t) => { + const configBlob = { + url: 'http://localhost:1234/', + timeout: 17, + retries: 19 + }; + + const libpostal = new Libpostal(configBlob); + + t.deepEquals(libpostal.getUrl(), 'http://localhost:1234/parse'); + t.end(); + + }); + +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`SERVICE CONFIGURATION /Libpostal ${name}`, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; From d4d52443dcbdde0e48daff5d68612a6e0dd3b00e Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 14 Nov 2017 11:02:33 -0500 Subject: [PATCH 05/61] Update Dockerfile for libpostal service The Dockerfile no longer needs to be build on the libpostal_baseimage (which will probably go away soon) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 041f00dc..e28228c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # base image -FROM pelias/libpostal_baseimage +FROM pelias/baseimage # maintainer information LABEL maintainer="pelias@mapzen.com" From 0a95dd25838fc8cf12d7aecbf01603a477cf25d2 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 14 Nov 2017 12:00:29 -0500 Subject: [PATCH 06/61] removed references to `textAnalyzer` config setting --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 4253d15e..fd42a7f6 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ The API recognizes the following properties under the top-level `api` key in you |parameter|required|default|description| |---|---|---|---| |`host`|*yes*||specifies the url under which the http service is to run| -|`textAnalyzer`|*no*|*addressit*|can be either `libpostal` or `addressit` however will soon be **deprecated** and only `libpostal` will be supported going forward| |`indexName`|*no*|*pelias*|name of the Elasticsearch index to be used when building queries| |`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable |`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.| @@ -66,7 +65,6 @@ Example configuration file would look something like this: "host": "localhost:3100/v1/", "indexName": "foobar", "relativeScores": true, - "textAnalyzer": "libpostal", "services": { "pip": { "url": "http://mypipservice.com:3000" From 51709b58ff24277e64193b5e9cc5cd2ff610a58f Mon Sep 17 00:00:00 2001 From: sweco-semhul Date: Thu, 16 Nov 2017 12:45:37 +0100 Subject: [PATCH 07/61] Updating ciao tests to reflect current status in master to make them run --- .../autocomplete/layers_alias_coarse.coffee | 5 +++- test/ciao/autocomplete/layers_invalid.coffee | 2 +- .../layers_mix_invalid_valid.coffee | 2 +- ...boundary_circle_valid_radius_coarse.coffee | 23 ++++++++++++++++--- test/ciao/reverse/layers_alias_coarse.coffee | 5 +++- test/ciao/reverse/layers_invalid.coffee | 2 +- .../reverse/layers_mix_invalid_valid.coffee | 2 +- test/ciao/reverse/sources_multiple.coffee | 4 ++-- test/ciao/search/layers_alias_coarse.coffee | 5 +++- test/ciao/search/layers_invalid.coffee | 2 +- .../search/layers_mix_invalid_valid.coffee | 2 +- 11 files changed, 40 insertions(+), 14 deletions(-) diff --git a/test/ciao/autocomplete/layers_alias_coarse.coffee b/test/ciao/autocomplete/layers_alias_coarse.coffee index 5ceaaa46..fe934f55 100644 --- a/test/ciao/autocomplete/layers_alias_coarse.coffee +++ b/test/ciao/autocomplete/layers_alias_coarse.coffee @@ -32,6 +32,7 @@ should.not.exist json.geocoding.warnings json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 json.geocoding.query.layers.should.eql [ "continent", + "empire", "country", "dependency", "macroregion", @@ -45,5 +46,7 @@ json.geocoding.query.layers.should.eql [ "continent", "neighbourhood", "microhood", "disputed", - "postalcode" + "postalcode", + "ocean", + "marinearea" ] diff --git a/test/ciao/autocomplete/layers_invalid.coffee b/test/ciao/autocomplete/layers_invalid.coffee index 5396387a..74b04379 100644 --- a/test/ciao/autocomplete/layers_invalid.coffee +++ b/test/ciao/autocomplete/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/autocomplete/layers_mix_invalid_valid.coffee b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee index 6a9cc488..7357fda5 100644 --- a/test/ciao/autocomplete/layers_mix_invalid_valid.coffee +++ b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/boundary_circle_valid_radius_coarse.coffee b/test/ciao/reverse/boundary_circle_valid_radius_coarse.coffee index b3f76c69..7d5d9bfc 100644 --- a/test/ciao/reverse/boundary_circle_valid_radius_coarse.coffee +++ b/test/ciao/reverse/boundary_circle_valid_radius_coarse.coffee @@ -26,14 +26,31 @@ json.features.should.be.instanceof Array should.not.exist json.geocoding.errors #? expected warnings -should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql [ 'boundary.circle.radius is not applicable for coarse reverse' ] +should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query['layers'].should.eql 'coarse' json.geocoding.query['point.lat'].should.eql 40.744243 json.geocoding.query['point.lon'].should.eql -73.990342 json.geocoding.query['boundary.circle.lat'].should.eql 40.744243 json.geocoding.query['boundary.circle.lon'].should.eql -73.990342 json.geocoding.query['boundary.circle.radius'].should.eql 999.9 +json.geocoding.query['layers'].should.eql [ "continent", + "empire", + "country", + "dependency", + "macroregion", + "region", + "locality", + "localadmin", + "macrocounty", + "county", + "macrohood", + "borough", + "neighbourhood", + "microhood", + "disputed", + "postalcode", + "ocean", + "marinearea" +] \ No newline at end of file diff --git a/test/ciao/reverse/layers_alias_coarse.coffee b/test/ciao/reverse/layers_alias_coarse.coffee index f70eeb11..276ca93f 100644 --- a/test/ciao/reverse/layers_alias_coarse.coffee +++ b/test/ciao/reverse/layers_alias_coarse.coffee @@ -31,6 +31,7 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 json.geocoding.query.layers.should.eql [ "continent", + "empire", "country", "dependency", "macroregion", @@ -44,5 +45,7 @@ json.geocoding.query.layers.should.eql [ "continent", "neighbourhood", "microhood", "disputed", - "postalcode" + "postalcode", + "ocean", + "marinearea" ] diff --git a/test/ciao/reverse/layers_invalid.coffee b/test/ciao/reverse/layers_invalid.coffee index 7298285a..cbd46a89 100644 --- a/test/ciao/reverse/layers_invalid.coffee +++ b/test/ciao/reverse/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_mix_invalid_valid.coffee b/test/ciao/reverse/layers_mix_invalid_valid.coffee index c3165cf7..90d0ea41 100644 --- a/test/ciao/reverse/layers_mix_invalid_valid.coffee +++ b/test/ciao/reverse/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/sources_multiple.coffee b/test/ciao/reverse/sources_multiple.coffee index 4fd24366..986adc0e 100644 --- a/test/ciao/reverse/sources_multiple.coffee +++ b/test/ciao/reverse/sources_multiple.coffee @@ -1,6 +1,6 @@ #> sources filter -path: '/v1/reverse?point.lat=1&point.lon=2&sources=openstreetmap,geonames' +path: '/v1/reverse?point.lat=1&point.lon=2&sources=openstreetmap,openaddresses' #? 200 ok response.statusCode.should.be.equal 200 @@ -30,4 +30,4 @@ should.not.exist json.geocoding.warnings #? inputs json.geocoding.query['size'].should.eql 10 -json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"] +json.geocoding.query.sources.should.eql ["openstreetmap", "openaddresses"] diff --git a/test/ciao/search/layers_alias_coarse.coffee b/test/ciao/search/layers_alias_coarse.coffee index af8d3d8e..c9ecf7d3 100644 --- a/test/ciao/search/layers_alias_coarse.coffee +++ b/test/ciao/search/layers_alias_coarse.coffee @@ -32,6 +32,7 @@ should.not.exist json.geocoding.warnings json.geocoding.query['text'].should.eql 'a' json.geocoding.query['size'].should.eql 10 json.geocoding.query.layers.should.eql [ "continent", + "empire", "country", "dependency", "macroregion", @@ -45,5 +46,7 @@ json.geocoding.query.layers.should.eql [ "continent", "neighbourhood", "microhood", "disputed", - "postalcode" + "postalcode", + "ocean", + "marinearea" ] diff --git a/test/ciao/search/layers_invalid.coffee b/test/ciao/search/layers_invalid.coffee index 52b370dd..2d8b6275 100644 --- a/test/ciao/search/layers_invalid.coffee +++ b/test/ciao/search/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_mix_invalid_valid.coffee b/test/ciao/search/layers_mix_invalid_valid.coffee index da39b9e8..91393881 100644 --- a/test/ciao/search/layers_mix_invalid_valid.coffee +++ b/test/ciao/search/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,dependency,macrocounty,macrohood,microhood,disputed,postalcode' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,street,country,macroregion,region,county,localadmin,locality,borough,neighbourhood,continent,empire,dependency,macrocounty,macrohood,microhood,disputed,postalcode,ocean,marinearea' ] #? expected warnings should.not.exist json.geocoding.warnings From c007245a5a531f1656e42282e1e90c39c192df5e Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 19 Jan 2018 21:43:27 +0000 Subject: [PATCH 08/61] chore(package): update source-map to version 0.7.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d32e37cc..8dae9e8d 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "precommit-hook": "^3.0.0", "proxyquire": "^1.7.10", "semantic-release": "^8.0.0", - "source-map": "^0.6.0", + "source-map": "^0.7.0", "tap-dot": "1.0.5", "tape": "^4.5.1", "tmp": "0.0.33", From ca66c0a3334a8dd6864ee18f7c721244b4f20c08 Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Sat, 27 Jan 2018 15:21:39 -0500 Subject: [PATCH 09/61] Dockerfile: Update to current Pelias contact email address Per https://github.com/pelias/pelias/tree/master/announcements/2018-01-02-pelias-update replace @mapzen.com email address in Dockerfile with the current Pelias contact email address. For example this will be visible from `docker inspect` with the other labels. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e28228c3..0f0eefe1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM pelias/baseimage # maintainer information -LABEL maintainer="pelias@mapzen.com" +LABEL maintainer="pelias.team@gmail.com" EXPOSE 3100 From 1dfc6988ed285891d5f043150b2b7f52c1d6622f Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Sat, 27 Jan 2018 15:24:41 -0500 Subject: [PATCH 10/61] Update to current Pelias contact email address Per https://github.com/pelias/pelias/tree/master/announcements/2018-01-02-pelias-update replace all @mapzen.com email addresses with the current Pelias contact email address. grep used to confirm no '@mapzen.com' remain. --- sanitizer/_deprecate_quattroshapes.js | 2 +- test/ciao/reverse/sources_deprecation_warning.coffee | 2 +- test/ciao/search/sources_deprecation_warning.coffee | 2 +- test/unit/sanitizer/_deprecate_quattroshapes.js | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sanitizer/_deprecate_quattroshapes.js b/sanitizer/_deprecate_quattroshapes.js index 50f3af8e..1db2fef1 100644 --- a/sanitizer/_deprecate_quattroshapes.js +++ b/sanitizer/_deprecate_quattroshapes.js @@ -26,7 +26,7 @@ function _sanitize( raw, clean, opts ) { 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'); + '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'); // user requested 'quattroshapes', we will give them 'whosonfirst' instead. sources = _.without(sources, 'quattroshapes', 'qs'); diff --git a/test/ciao/reverse/sources_deprecation_warning.coffee b/test/ciao/reverse/sources_deprecation_warning.coffee index 45c2c967..51445800 100644 --- a/test/ciao/reverse/sources_deprecation_warning.coffee +++ b/test/ciao/reverse/sources_deprecation_warning.coffee @@ -27,7 +27,7 @@ should.not.exist json.geocoding.errors #? expected warnings should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ] +json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] #? inputs json.geocoding.query['size'].should.eql 10 diff --git a/test/ciao/search/sources_deprecation_warning.coffee b/test/ciao/search/sources_deprecation_warning.coffee index c6d96a2c..fe82d298 100644 --- a/test/ciao/search/sources_deprecation_warning.coffee +++ b/test/ciao/search/sources_deprecation_warning.coffee @@ -27,7 +27,7 @@ should.not.exist json.geocoding.errors #? expected warnings should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email search@mapzen.com.' ] +json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] #? inputs json.geocoding.query['size'].should.eql 10 diff --git a/test/unit/sanitizer/_deprecate_quattroshapes.js b/test/unit/sanitizer/_deprecate_quattroshapes.js index d468b97f..6cbe1f09 100644 --- a/test/unit/sanitizer/_deprecate_quattroshapes.js +++ b/test/unit/sanitizer/_deprecate_quattroshapes.js @@ -15,7 +15,7 @@ module.exports.tests.warning_message_1 = function(test, common) { 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'] + '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] }, 'warning emitted'); t.end(); @@ -35,7 +35,7 @@ module.exports.tests.warning_message_2 = function(test, common) { 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email search@mapzen.com.'] + '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] }, 'warning emitted'); t.end(); From c2b86f305ad9378ad934b820008aa69ffd0d78fe Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Sat, 27 Jan 2018 16:12:57 -0500 Subject: [PATCH 11/61] Add MIT license pelias-api never previously had a LICENSE, although it appears clear from contextual information that the intention was for pelias-api to be MIT licensed (e.g. metadata in package.json to this effect). Copy over MIT license as formatted in the pelias/pelias meta repository. --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d2a640a9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mapzen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From 466eb12a615e93fd45b6cb4c8744353825059089 Mon Sep 17 00:00:00 2001 From: Rhys Kidd Date: Mon, 29 Jan 2018 21:27:19 -0500 Subject: [PATCH 12/61] Update to current upstream Who's On First URL grep used to confirm no other direct whosonfirst.mapzen.com links remain. --- public/attribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/attribution.md b/public/attribution.md index 0eedb365..009450f6 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -4,4 +4,4 @@ * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/) - * [WhosOnFirst](http://whosonfirst.mapzen.com) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md) + * [WhosOnFirst](https://www.whosonfirst.org/) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md) From 533cb4b39e30ca6cac519f29b426f83d409d3c23 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 1 Feb 2018 21:37:58 +0000 Subject: [PATCH 13/61] fix(package): update pelias-config to version 2.14.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d32e37cc..9f9e9f73 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "markdown": "0.5.0", "morgan": "^1.8.2", "pelias-categories": "1.2.0", - "pelias-config": "2.13.0", + "pelias-config": "2.14.0", "pelias-labels": "1.7.0", "pelias-logger": "0.3.0", "pelias-microservice-wrapper": "1.3.0", From 8f4069d91c8d628c1178a3e11ce5cdfbdd16b7cc Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 1 Feb 2018 22:34:06 +0000 Subject: [PATCH 14/61] fix(package): update pelias-model to version 5.3.2 Closes #1065 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8dae9e8d..aaada149 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "pelias-labels": "1.7.0", "pelias-logger": "0.3.0", "pelias-microservice-wrapper": "1.3.0", - "pelias-model": "5.2.0", + "pelias-model": "5.3.2", "pelias-query": "9.1.1", "pelias-sorting": "1.1.0", "predicates": "^2.0.0", From b8494ae41b7f9f41057b9d86c1afe2b43c13bb28 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 2 Feb 2018 04:29:52 +0000 Subject: [PATCH 15/61] fix(package): update pelias-logger to version 0.3.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bebc79b3..ea455117 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "pelias-categories": "1.2.0", "pelias-config": "2.14.0", "pelias-labels": "1.7.0", - "pelias-logger": "0.3.0", + "pelias-logger": "0.3.1", "pelias-microservice-wrapper": "1.3.0", "pelias-model": "5.3.2", "pelias-query": "9.1.1", From d187e7d950b00c75fef4b6d5c2118e25b3da6328 Mon Sep 17 00:00:00 2001 From: Joxit Date: Sun, 21 Jan 2018 16:38:40 +0100 Subject: [PATCH 16/61] Fix for #1077, fail to search `Saint..` cities in structured queries --- sanitizer/_city_name_standardizer.js | 16 ++++++++-------- test/unit/sanitizer/_city_name_standardizer.js | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sanitizer/_city_name_standardizer.js b/sanitizer/_city_name_standardizer.js index 6724eda1..236c0df7 100644 --- a/sanitizer/_city_name_standardizer.js +++ b/sanitizer/_city_name_standardizer.js @@ -1,13 +1,13 @@ const _ = require('lodash'); // matches 'ft', 'mt', 'saint', and 'sainte' on word boundary -const mountSaintFort = /\b([fm]t|sainte?)\b/g; +const mountSaintFort = /\b([fm]t|ste?)\b/g; const transliterations = { 'mt': 'mount', 'ft': 'fort', - 'saint': 'st', - 'sainte': 'ste' + 'st': 'saint', + 'ste': 'sainte' }; function transliterate(match) { @@ -22,15 +22,15 @@ function _sanitize(raw, clean) { // only try to transliterate if there is a city in parsed_text if (!_.isEmpty(_.get(clean, 'parsed_text.city'))) { - // eg input: Ft. Saint Louis - // after 1. ft saint louis - // after 2. fort st louis - // after 3. fort st louis + // eg input: Ft. st Louis + // after 1. ft st louis + // after 2. fort saint louis + // after 3. fort saint louis // 1. remove '.' that could abbreviate ft and mt (makes transliteration regex easier) const periods_removed = _.toLower(clean.parsed_text.city).replace(/\b(mt|ft)\./g, '$1 '); - // 2. transliterate 'saint'->'st', etc + // 2. transliterate 'st'->'saint', etc const transliterated = periods_removed.replace(mountSaintFort, transliterate); // 3. reduce whitespace sequences that can occur when removing periods down to a single space diff --git a/test/unit/sanitizer/_city_name_standardizer.js b/test/unit/sanitizer/_city_name_standardizer.js index 5a01e58b..9ae5917e 100644 --- a/test/unit/sanitizer/_city_name_standardizer.js +++ b/test/unit/sanitizer/_city_name_standardizer.js @@ -48,7 +48,7 @@ module.exports.tests.text_parser = function(test, common) { }); - test('\'saint\' should be abbreviated to \'st\' wherever it appears in the city', function(t) { + test('\'st\' should be expanded to \'saint\' wherever it appears in the city', function(t) { const raw = {}; const clean = { @@ -56,7 +56,7 @@ module.exports.tests.text_parser = function(test, common) { query: 'saint query value', neighbourhood: 'saint neighbourhood value', borough: 'saint borough value', - city: 'SainT city sAiNt value saInt', + city: 'st city ST value St', county: 'saint county value', state: 'saint state value', postalcode: 'saint postalcode value', @@ -69,7 +69,7 @@ module.exports.tests.text_parser = function(test, common) { query: 'saint query value', neighbourhood: 'saint neighbourhood value', borough: 'saint borough value', - city: 'st city st value st', + city: 'saint city saint value saint', county: 'saint county value', state: 'saint state value', postalcode: 'saint postalcode value', @@ -86,7 +86,7 @@ module.exports.tests.text_parser = function(test, common) { }); - test('\'sainte\' should be abbreviated to \'ste\' wherever it appears in the city', function(t) { + test('\'ste\' should be expanded to \'sainte\' wherever it appears in the city', function(t) { const raw = {}; const clean = { @@ -94,7 +94,7 @@ module.exports.tests.text_parser = function(test, common) { query: 'sainte query value', neighbourhood: 'sainte neighbourhood value', borough: 'sainte borough value', - city: 'SaintE city sAinTe value saINte', + city: 'ste city STE value StE', county: 'sainte county value', state: 'sainte state value', postalcode: 'sainte postalcode value', @@ -107,7 +107,7 @@ module.exports.tests.text_parser = function(test, common) { query: 'sainte query value', neighbourhood: 'sainte neighbourhood value', borough: 'sainte borough value', - city: 'ste city ste value ste', + city: 'sainte city sainte value sainte', county: 'sainte county value', state: 'sainte state value', postalcode: 'sainte postalcode value', @@ -200,18 +200,18 @@ module.exports.tests.text_parser = function(test, common) { }); - test('mixture of \'mt\', \'ft\', \'saint\', and \'sainte\' should be expanded/abbreviated', function(t) { + test('mixture of \'mt\', \'ft\', \'st\', and \'st\' should be expanded', function(t) { const raw = {}; const clean = { parsed_text: { - city: 'mt. ft saint sainte mt ft.' + city: 'mt. ft st ste mt ft.' } }; const expected_clean = { parsed_text: { - city: 'mount fort st ste mount fort' + city: 'mount fort saint sainte mount fort' } }; From 551f0227dbe16827aa92f28a44c0f1a76ac4343a Mon Sep 17 00:00:00 2001 From: Joxit Date: Wed, 28 Feb 2018 13:52:56 +0100 Subject: [PATCH 17/61] Transliterates Saint and Sainte in ES schema see pelias/schema#268 --- sanitizer/_city_name_standardizer.js | 18 ++-- .../unit/sanitizer/_city_name_standardizer.js | 82 +------------------ 2 files changed, 11 insertions(+), 89 deletions(-) diff --git a/sanitizer/_city_name_standardizer.js b/sanitizer/_city_name_standardizer.js index 236c0df7..e4604474 100644 --- a/sanitizer/_city_name_standardizer.js +++ b/sanitizer/_city_name_standardizer.js @@ -1,20 +1,18 @@ const _ = require('lodash'); -// matches 'ft', 'mt', 'saint', and 'sainte' on word boundary -const mountSaintFort = /\b([fm]t|ste?)\b/g; +// matches 'ft', 'mt' on word boundary +const mountFort = /\b([fm]t)\b/g; const transliterations = { 'mt': 'mount', - 'ft': 'fort', - 'st': 'saint', - 'ste': 'sainte' + 'ft': 'fort' }; function transliterate(match) { return _.get(transliterations, match); } -// transliterate ft/mt/saint/sainte to fort/mount/st/ste, respectively +// transliterate ft/mt to fort/mount, respectively function _sanitize(raw, clean) { // error & warning messages // this function doesn't add any error or warning messages @@ -24,14 +22,14 @@ function _sanitize(raw, clean) { if (!_.isEmpty(_.get(clean, 'parsed_text.city'))) { // eg input: Ft. st Louis // after 1. ft st louis - // after 2. fort saint louis - // after 3. fort saint louis + // after 2. fort st louis + // after 3. fort st louis // 1. remove '.' that could abbreviate ft and mt (makes transliteration regex easier) const periods_removed = _.toLower(clean.parsed_text.city).replace(/\b(mt|ft)\./g, '$1 '); - // 2. transliterate 'st'->'saint', etc - const transliterated = periods_removed.replace(mountSaintFort, transliterate); + // 2. transliterate 'ft'->'fort', etc + const transliterated = periods_removed.replace(mountFort, transliterate); // 3. reduce whitespace sequences that can occur when removing periods down to a single space const whitespace_normalized = _.trimEnd(transliterated.replace(/\s+/, ' ')); diff --git a/test/unit/sanitizer/_city_name_standardizer.js b/test/unit/sanitizer/_city_name_standardizer.js index 9ae5917e..9e168085 100644 --- a/test/unit/sanitizer/_city_name_standardizer.js +++ b/test/unit/sanitizer/_city_name_standardizer.js @@ -48,82 +48,6 @@ module.exports.tests.text_parser = function(test, common) { }); - test('\'st\' should be expanded to \'saint\' wherever it appears in the city', function(t) { - const raw = {}; - - const clean = { - parsed_text: { - query: 'saint query value', - neighbourhood: 'saint neighbourhood value', - borough: 'saint borough value', - city: 'st city ST value St', - county: 'saint county value', - state: 'saint state value', - postalcode: 'saint postalcode value', - country: 'saint country value' - } - }; - - const expected_clean = { - parsed_text: { - query: 'saint query value', - neighbourhood: 'saint neighbourhood value', - borough: 'saint borough value', - city: 'saint city saint value saint', - county: 'saint county value', - state: 'saint state value', - postalcode: 'saint postalcode value', - country: 'saint country value' - } - }; - - const messages = sanitizer.sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('\'ste\' should be expanded to \'sainte\' wherever it appears in the city', function(t) { - const raw = {}; - - const clean = { - parsed_text: { - query: 'sainte query value', - neighbourhood: 'sainte neighbourhood value', - borough: 'sainte borough value', - city: 'ste city STE value StE', - county: 'sainte county value', - state: 'sainte state value', - postalcode: 'sainte postalcode value', - country: 'sainte country value' - } - }; - - const expected_clean = { - parsed_text: { - query: 'sainte query value', - neighbourhood: 'sainte neighbourhood value', - borough: 'sainte borough value', - city: 'sainte city sainte value sainte', - county: 'sainte county value', - state: 'sainte state value', - postalcode: 'sainte postalcode value', - country: 'sainte country value' - } - }; - - const messages = sanitizer.sanitize(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - test('\'ft\' should be expanded to \'fort\' wherever it appears in the city', function(t) { const raw = {}; @@ -200,18 +124,18 @@ module.exports.tests.text_parser = function(test, common) { }); - test('mixture of \'mt\', \'ft\', \'st\', and \'st\' should be expanded', function(t) { + test('mixture of \'mt\', \'ft\' should be expanded', function(t) { const raw = {}; const clean = { parsed_text: { - city: 'mt. ft st ste mt ft.' + city: 'mt. ft mt ft.' } }; const expected_clean = { parsed_text: { - city: 'mount fort saint sainte mount fort' + city: 'mount fort mount fort' } }; From 3489517da3670054c9ab4b480bc66b8f5485e8d0 Mon Sep 17 00:00:00 2001 From: antoine-de Date: Mon, 4 Dec 2017 18:01:18 +0100 Subject: [PATCH 18/61] dedup osm/OA address linked to https://github.com/pelias/pelias/issues/541 and maybe to https://github.com/pelias/pelias/issues/541 but I cannot reproduce it. and to the fact that there are lots of dupplicates in france like: [20 rue hector malot paris](https://mapzen.com/search/explorer/?query=search&text=20%20rue%20hector%20malot%20paris) that returns 1 result from open address and 1 result from OSM now we first check which source has a zipcode and if both have, we prefer OA over OSM --- middleware/dedupe.js | 10 +++++----- test/unit/middleware/dedupe.js | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 129d157e..a50b0494 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -64,13 +64,13 @@ function dedupeResults(req, res, next) { function isPreferred(existing, candidateReplacement) { // NOTE: we are assuming here that the layer for both records is the same - - var isOA = _.flow(_.property('source'), _.eq.bind(null, 'openaddresses')); - var hasZip = _.bind(_.has, null, _.bind.placeholder, 'address_parts.zip'); + const hasZip = _.bind(_.has, null, _.bind.placeholder, 'address_parts.zip'); // https://github.com/pelias/api/issues/872 - if (isOA(existing) && isOA(candidateReplacement)) { - return hasZip(candidateReplacement) && !hasZip(existing); + const candidateHasZip = hasZip(candidateReplacement); + const existingHasZip = hasZip(existing); + if (candidateHasZip !== existingHasZip) { + return candidateHasZip; } //bind the trumps function to the data items to keep the rest of the function clean diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js index c3821035..ca8a5cae 100644 --- a/test/unit/middleware/dedupe.js +++ b/test/unit/middleware/dedupe.js @@ -222,6 +222,42 @@ module.exports.tests.trump = function(test, common) { t.end(); }); }); + +test('osm with zip trumps openaddresses without zip', function (t) { + var req = { + clean: { + text: '100 Main St', + size: 100 + } + }; + var res = { + data: [ + { + 'name': { 'default': '100 Main St' }, + 'source': 'openaddresses', + 'source_id': '123456', + 'layer': 'address', + 'address_parts': {} + }, + { + 'name': { 'default': '100 Main St' }, + 'source': 'openstreetmap', + 'source_id': '654321', + 'layer': 'address', + 'address_parts': { + 'zip': '54321' + } + } + ] + }; + + var expectedCount = 1; + dedupe(req, res, function () { + t.equal(res.data.length, expectedCount, 'results have fewer items than before'); + t.deepEqual(res.data[0].source_id, '654321', 'openstreetmap result with zip won'); + t.end(); + }); +}); }; module.exports.all = function (tape, common) { From a3e8ae918b92f773990acdd77c1c440cc4f5eecb Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:39:19 -0400 Subject: [PATCH 19/61] Rewrite Dockerfile There were a couple problems with the current dockerfile: * It set the userid of the processes running in the container to 9999, without creating a user with that ID. This leads to confusion and an annoying message when you run an interactive bash session (the shell PS1 would display something like `I have no name!@1438586f786e:~$` * It tried to run `chown` on _all_ code files after running NPM install. This takes a really long time * It did not copy `package.json` and run `npm install` before copying other files. This means even a one line code change causes the image rebuild process to re-run `npm install`, which takes 30 seconds or so Now the image creates and uses a pelias user, sets permissions correctly from the start to avoid `chown`, and only runs `npm install` when it absolutely has to. --- Dockerfile | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f0eefe1..bd91cd16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # base image FROM pelias/baseimage +RUN useradd -ms /bin/bash pelias +USER pelias # maintainer information LABEL maintainer="pelias.team@gmail.com" @@ -7,23 +9,17 @@ LABEL maintainer="pelias.team@gmail.com" EXPOSE 3100 # Where the app is built and run inside the docker fs -ENV WORK=/opt/pelias +ENV WORK=/home/pelias +WORKDIR ${WORK} -# Used indirectly for saving npm logs etc. -ENV HOME=/opt/pelias +# copy package.json first to prevent npm install being rerun when only code changes +COPY ./package.json ${WORK} +RUN npm install -WORKDIR ${WORK} COPY . ${WORK} -# Build and set permissions for arbitrary non-root user -RUN npm install && \ - npm test && \ - chmod -R a+rwX . - -# Don't run as root, because there's no reason to (https://docs.docker.com/engine/articles/dockerfile_best-practices/#user). -# This also reveals permission problems on local Docker. -RUN chown -R 9999:9999 ${WORK} -USER 9999 +# only allow containers to succeed if tests pass +RUN npm test # start service CMD [ "npm", "start" ] From 07cabd8a32c95042a07d40d53545c3e5070bfe46 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 9 Mar 2018 18:15:14 -0500 Subject: [PATCH 20/61] fix: Avoid querying PIP service for non-coarse reverse Queries that specified only non-corase layers (address or venue) and had no results returned from Elasticsearch would trigger a request to the PIP service. The PIP service does not contain any addresses or venues so this query will never return anything, and only waste time. --- routes/v1.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/v1.js b/routes/v1.js index fb3539a8..f4e8176f 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -138,7 +138,7 @@ function addRoutes(app, peliasConfig) { // fallback to coarse reverse when regular reverse didn't return anything const coarseReverseShouldExecute = all( - isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData) + isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData), not(isOnlyNonAdminLayers) ); const libpostalShouldExecute = all( From 2e92f15a918adbd3904cb33e4c4715df5a544f54 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:51:44 -0400 Subject: [PATCH 21/61] Whitespace --- middleware/sizeCalculator.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index 69409e45..f8c6936c 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -8,15 +8,15 @@ var MIN_QUERY_SIZE = 20; * Utility for calculating query result size * incorporating padding for dedupe process */ -function setup() { - return function setQuerySize(req, res, next) { - if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { - return next(); - } +function setup(min_size) { + return function setQuerySize(req, res, next) { + if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { + return next(); + } - req.clean.querySize = calculateSize(req.clean.size); - next(); - }; + req.clean.querySize = calculateSize(req.clean.size, min_size); + next(); + }; } /** From c966e6f6aed5c5ce6126413258024782301a434c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:52:01 -0400 Subject: [PATCH 22/61] Allow the minimum Elasticearch query size to be set per endpoint Reverse queries don't need as much deduping, for example --- middleware/sizeCalculator.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index f8c6936c..9b577916 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -9,6 +9,10 @@ var MIN_QUERY_SIZE = 20; * incorporating padding for dedupe process */ function setup(min_size) { + if (min_size === undefined) { + min_size = MIN_QUERY_SIZE; + } + return function setQuerySize(req, res, next) { if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { return next(); @@ -25,8 +29,8 @@ function setup(min_size) { * @param {number} cleanSize * @returns {number} */ -function calculateSize(cleanSize) { - return Math.max(MIN_QUERY_SIZE, cleanSize * SIZE_PADDING); +function calculateSize(cleanSize, min_size) { + return Math.max(min_size, cleanSize * SIZE_PADDING); } module.exports = setup; From 07dd704e1d946ebc2ceb020878dd5b6157806542 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:52:27 -0400 Subject: [PATCH 23/61] Set ES query minimum size to 2 for /v1/reverse --- routes/v1.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/v1.js b/routes/v1.js index f4e8176f..6b40f1d6 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -344,7 +344,7 @@ function addRoutes(app, peliasConfig) { reverse: createRouter([ sanitizers.reverse.middleware, middleware.requestLanguage, - middleware.calcSize(), + middleware.calcSize(2), controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute), controllers.coarse_reverse(pipService, coarseReverseShouldExecute), postProc.distances('point.'), From 5247ea575daf5dcfd5be222583892b89c3db595c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:15:27 -0400 Subject: [PATCH 24/61] Give variable a better name --- middleware/sizeCalculator.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index 9b577916..f7ea906a 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -2,7 +2,7 @@ var _ = require('lodash'); var SIZE_PADDING = 2; -var MIN_QUERY_SIZE = 20; +var DEFAULT_MIN_QUERY_SIZE = 20; /** * Utility for calculating query result size @@ -10,7 +10,7 @@ var MIN_QUERY_SIZE = 20; */ function setup(min_size) { if (min_size === undefined) { - min_size = MIN_QUERY_SIZE; + min_size = DEFAULT_MIN_QUERY_SIZE; } return function setQuerySize(req, res, next) { From 825b506a7522473ba422af0711d60ab2b01a7785 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:22:08 -0400 Subject: [PATCH 25/61] Revise tests to account for sizeCalculator setup params --- test/unit/helper/sizeCalculator.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index 6c8d8d22..cbcd91b8 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -1,11 +1,10 @@ - -var calcSize = require('../../../middleware/sizeCalculator.js')(); +var calcSizeMiddleware = require('../../../middleware/sizeCalculator.js'); module.exports.tests = {}; module.exports.tests.interface = function(test, common) { test('interface', function(t) { - t.equal(typeof calcSize, 'function', 'valid function'); + t.equal(typeof calcSizeMiddleware, 'function', 'valid function'); t.end(); }); }; @@ -24,6 +23,7 @@ module.exports.tests.valid = function(test, common) { test('size=0', function (t) { setup(0); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -32,6 +32,7 @@ module.exports.tests.valid = function(test, common) { test('size=1', function (t) { setup(1); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -40,6 +41,7 @@ module.exports.tests.valid = function(test, common) { test('size=10', function (t) { setup(10); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -48,6 +50,7 @@ module.exports.tests.valid = function(test, common) { test('size=20', function (t) { setup(20); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 40); t.end(); @@ -56,6 +59,7 @@ module.exports.tests.valid = function(test, common) { test('no size', function (t) { setup(); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); t.end(); From 94946989bb4c872085de452f10a5ed3a98409113 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:24:59 -0400 Subject: [PATCH 26/61] Test cases with a lower min query size --- test/unit/helper/sizeCalculator.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index cbcd91b8..fe4f9bb3 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -65,6 +65,33 @@ module.exports.tests.valid = function(test, common) { t.end(); }); }); + + test('no size, min query size 10', function (t) { + setup(); + const calcSize = calcSizeMiddleware(10); + calcSize(req, {}, function () { + t.equal(req.clean.hasOwnProperty('querySize'), false); + t.end(); + }); + }); + + test('size 5, min query size 10', function (t) { + setup(5); + const calcSize = calcSizeMiddleware(10); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 10); + t.end(); + }); + }); + + test('size 3, min query size 2', function (t) { + setup(3); + const calcSize = calcSizeMiddleware(2); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 6); + t.end(); + }); + }); }; module.exports.all = function (tape, common) { From f887d7a2499972f7c6da6277df8b615ba53b6fdf Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:26:57 -0400 Subject: [PATCH 27/61] Rename test setup function It now can be confused for the middleware setup function --- test/unit/helper/sizeCalculator.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index fe4f9bb3..204c6870 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -11,7 +11,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.valid = function(test, common) { var req = { clean: {} }; - function setup(val) { + function setupQuery(val) { if (isNaN(val)) { delete req.clean.size; } @@ -22,7 +22,7 @@ module.exports.tests.valid = function(test, common) { } test('size=0', function (t) { - setup(0); + setupQuery(0); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -31,7 +31,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=1', function (t) { - setup(1); + setupQuery(1); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -40,7 +40,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=10', function (t) { - setup(10); + setupQuery(10); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -49,7 +49,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=20', function (t) { - setup(20); + setupQuery(20); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 40); @@ -58,7 +58,7 @@ module.exports.tests.valid = function(test, common) { }); test('no size', function (t) { - setup(); + setupQuery(); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); @@ -67,7 +67,7 @@ module.exports.tests.valid = function(test, common) { }); test('no size, min query size 10', function (t) { - setup(); + setupQuery(); const calcSize = calcSizeMiddleware(10); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); @@ -76,7 +76,7 @@ module.exports.tests.valid = function(test, common) { }); test('size 5, min query size 10', function (t) { - setup(5); + setupQuery(5); const calcSize = calcSizeMiddleware(10); calcSize(req, {}, function () { t.equal(req.clean.querySize, 10); @@ -85,7 +85,7 @@ module.exports.tests.valid = function(test, common) { }); test('size 3, min query size 2', function (t) { - setup(3); + setupQuery(3); const calcSize = calcSizeMiddleware(2); calcSize(req, {}, function () { t.equal(req.clean.querySize, 6); From dccad99d78518e727ed6ad7367cee717989d0e0b Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 14:53:10 -0400 Subject: [PATCH 28/61] Move sizeCalculator middleware tests to the correct place --- test/unit/{helper => middleware}/sizeCalculator.js | 0 test/unit/run.js | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/unit/{helper => middleware}/sizeCalculator.js (100%) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/middleware/sizeCalculator.js similarity index 100% rename from test/unit/helper/sizeCalculator.js rename to test/unit/middleware/sizeCalculator.js diff --git a/test/unit/run.js b/test/unit/run.js index ffebf9ea..1a557733 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -36,7 +36,6 @@ var tests = [ require('./helper/geojsonify'), require('./helper/logging'), require('./helper/type_mapping'), - require('./helper/sizeCalculator'), require('./helper/stackTraceLine'), require('./middleware/access_log'), require('./middleware/accuracy'), @@ -52,6 +51,7 @@ var tests = [ require('./middleware/parseBBox'), require('./middleware/sendJSON'), require('./middleware/normalizeParentIds'), + require('./middleware/sizeCalculator'), require('./middleware/sortResponseData'), require('./middleware/trimByGranularity'), require('./middleware/trimByGranularityStructured'), From a8a6991b2626e46df359dc1ea4df86cc576e9660 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 12 Mar 2018 09:12:57 -0400 Subject: [PATCH 29/61] Avoid NPM when starting Dockerfile --- Dockerfile | 2 +- bin/start | 2 ++ package.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100755 bin/start diff --git a/Dockerfile b/Dockerfile index bd91cd16..aa21a067 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,4 @@ COPY . ${WORK} RUN npm test # start service -CMD [ "npm", "start" ] +CMD [ "./bin/start" ] diff --git a/bin/start b/bin/start new file mode 100755 index 00000000..84794d53 --- /dev/null +++ b/bin/start @@ -0,0 +1,2 @@ +#!/bin/bash +exec node index.js diff --git a/package.json b/package.json index ea455117..427f7a17 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "docs": "./bin/generate-docs", "lint": "jshint .", - "start": "node index.js", + "start": "./bin/start", "test": "npm run unit", "travis": "npm run check-dependencies && npm test", "unit": "./bin/units", From 1752875f11a529d1c6cea5f7613b97c01bb65806 Mon Sep 17 00:00:00 2001 From: missinglink Date: Thu, 22 Mar 2018 15:23:56 +0100 Subject: [PATCH 30/61] feat: fix attribution link --- middleware/geocodeJSON.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/middleware/geocodeJSON.js b/middleware/geocodeJSON.js index 683feac4..0ff2e303 100644 --- a/middleware/geocodeJSON.js +++ b/middleware/geocodeJSON.js @@ -47,7 +47,11 @@ function convertToGeocodeJSON(req, res, next, opts) { // OPTIONAL. Default: null. The attribution of the data. In case of multiple sources, // and then multiple attributions, can be an object with one key by source. // Can be a URI on the server, which outlines attribution details. - res.body.geocoding.attribution = url.resolve(opts.config.host, opts.basePath + 'attribution'); + res.body.geocoding.attribution = url.format({ + protocol: req.protocol, + host: req.get('host'), + pathname: opts.basePath + 'attribution' + }); // OPTIONAL. Default: null. The query that has been issued to trigger the // search. From 38c082357b5d654c84398a42af83983ba526e347 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 22 Mar 2018 12:11:19 -0400 Subject: [PATCH 31/61] Remove EXPOSE command from Dockerfile Since we use the `PORT` env var to configure the port, this value will often be out of date. It leads to some confusing output in, for example `docker-compose ps` when using a port other than the default: ``` julian@manhattan ~/repos/pelias/dockerfiles $ docker-compose ps Name Command State Ports --------------------------------------------------------------------------------------------------------------------- pelias_api ./bin/start Up 3100/tcp, 0.0.0.0:4000->4000/tcp ``` Notice how the Ports section shows 3100, even though nothing is running on that port in the container, because the `PORT` env var was set to 4000. --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa21a067..dc7836e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,6 @@ USER pelias # maintainer information LABEL maintainer="pelias.team@gmail.com" -EXPOSE 3100 - # Where the app is built and run inside the docker fs ENV WORK=/home/pelias WORKDIR ${WORK} From 152e7cf7c4d29540414ed0a51d445bb5063b7d39 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 16 Mar 2018 20:35:07 +0000 Subject: [PATCH 32/61] fix(package): update elasticsearch to version 14.2.1 Closes #1062 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 427f7a17..e94c667b 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "addressit": "1.5.0", "async": "^2.0.0", "check-types": "^7.0.0", - "elasticsearch": "^13.0.0", + "elasticsearch": "^14.2.1", "elasticsearch-exceptions": "0.0.4", "express": "^4.8.8", "geojson": "^0.5.0", From e5e3b92af37f2606a8dba65e686a0edcf7c253da Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sat, 24 Mar 2018 14:26:17 -0400 Subject: [PATCH 33/61] Fix extremely verbose, repeated error output In some error cases, a warning is repeated many many times. It turns out there is code checking against elasticsearch error codes, and warning _each_ time it fails to match aganst one of them. Many times, the error that is being compared is not from elasticsearch, and in any case, the [elasticsearch-exceptions](https://www.npmjs.com/package/elasticsearch-exceptions) module is 3 years old. We should rewrite most of this code and stop using that module. For now, this at least reduces log noise. --- middleware/sendJSON.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/sendJSON.js b/middleware/sendJSON.js index 906145ed..584fdf0e 100644 --- a/middleware/sendJSON.js +++ b/middleware/sendJSON.js @@ -59,8 +59,8 @@ function sendJSONResponse(req, res, next) { statusCode = Math.max( statusCode, 500 ); break; // break on first match } - logger.warn( 'unknown geocoding error string:', err ); } + logger.warn( 'unknown geocoding error string:', err ); } else { logger.warn( 'unknown geocoding error type:', typeof err, err ); } From 7a01e794cfd3582df46dd088adbabf89c1396aea Mon Sep 17 00:00:00 2001 From: missinglink Date: Thu, 22 Mar 2018 13:39:40 +0100 Subject: [PATCH 34/61] support aliases for name fields --- helper/fieldValue.js | 47 ++++++++++ helper/geojsonify.js | 3 +- helper/geojsonify_place_details.js | 36 +------- middleware/confidenceScore.js | 11 +-- middleware/dedupe.js | 11 +-- middleware/localNamingConventions.js | 7 +- test/unit/helper/diffPlaces.js | 16 ++++ test/unit/helper/fieldValue.js | 41 +++++++++ test/unit/helper/geojsonify.js | 45 ++++++++++ test/unit/helper/geojsonify_place_details.js | 2 +- test/unit/middleware/confidenceScore.js | 32 +++++++ test/unit/middleware/dedupe.js | 90 ++++++++++++------- .../unit/middleware/localNamingConventions.js | 4 +- test/unit/run.js | 1 + 14 files changed, 266 insertions(+), 80 deletions(-) create mode 100644 helper/fieldValue.js create mode 100644 test/unit/helper/fieldValue.js diff --git a/helper/fieldValue.js b/helper/fieldValue.js new file mode 100644 index 00000000..147bf625 --- /dev/null +++ b/helper/fieldValue.js @@ -0,0 +1,47 @@ +'use strict'; + +const _ = require('lodash'); + +function getStringValue(property) { + // numeric value, cast to string + if (_.isNumber(property)) { + return _.toString(property); + } + + // isEmpty check works for all types of values: strings, arrays, objects + if (_.isEmpty(property)) { + return ''; + } + + if (_.isString(property)) { + return property; + } + + // array value, take first item in array (at this time only used for admin values) + if (_.isArray(property)) { + return property[0]; + } + + return _.toString(property); +} + +function getArrayValue(property) { + // numeric value, cast to array + if (_.isNumber(property)) { + return [property]; + } + + // isEmpty check works for all types of values: strings, arrays, objects + if (_.isEmpty(property)) { + return []; + } + + if (_.isArray(property)) { + return property; + } + + return [property]; +} + +module.exports.getStringValue = getStringValue; +module.exports.getArrayValue = getArrayValue; diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 515446ce..5c571fa7 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -5,6 +5,7 @@ const logger = require('pelias-logger').get('geojsonify'); const collectDetails = require('./geojsonify_place_details'); const _ = require('lodash'); const Document = require('pelias-model').Document; +const field = require('./fieldValue'); function geojsonifyPlaces( params, docs ){ @@ -53,7 +54,7 @@ function geojsonifyPlace(params, place) { // assign name, logging a warning if it doesn't exist if (_.has(place, 'name.default')) { - doc.name = place.name.default; + doc.name = field.getStringValue(place.name.default); } else { logger.warn(`doc ${doc.gid} does not contain name.default`); } diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js index c80da8b1..4c2b60b7 100644 --- a/helper/geojsonify_place_details.js +++ b/helper/geojsonify_place_details.js @@ -1,6 +1,7 @@ 'use strict'; const _ = require('lodash'); +const field = require('./fieldValue'); // Properties to be copied // If a property is identified as a single string, assume it should be presented as a string in response @@ -89,10 +90,10 @@ function collectProperties( params, source ) { switch (prop.type) { case 'string': - value = getStringValue(source[prop.name]); + value = field.getStringValue(source[prop.name]); break; case 'array': - value = getArrayValue(source[prop.name]); + value = field.getArrayValue(source[prop.name]); break; // default behavior is to copy property exactly as is default: @@ -110,35 +111,4 @@ function collectProperties( params, source ) { } -function getStringValue(property) { - // isEmpty check works for all types of values: strings, arrays, objects - if (_.isEmpty(property)) { - return ''; - } - - if (_.isString(property)) { - return property; - } - - // array value, take first item in array (at this time only used for admin values) - if (_.isArray(property)) { - return property[0]; - } - - return _.toString(property); -} - -function getArrayValue(property) { - // isEmpty check works for all types of values: strings, arrays, objects - if (_.isEmpty(property)) { - return ''; - } - - if (_.isArray(property)) { - return property; - } - - return [property]; -} - module.exports = collectProperties; diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js index 5f82ddf6..49597251 100644 --- a/middleware/confidenceScore.js +++ b/middleware/confidenceScore.js @@ -11,9 +11,10 @@ * - detection (or specification) of query type. i.e. an address shouldn't match an admin address. */ -var stats = require('stats-lite'); -var logger = require('pelias-logger').get('api'); -var check = require('check-types'); +const stats = require('stats-lite'); +const logger = require('pelias-logger').get('api'); +const check = require('check-types'); +const field = require('../helper/fieldValue'); var RELATIVE_SCORES = true; @@ -131,12 +132,12 @@ function checkDistanceFromMean(score, mean, stdev) { function checkName(text, parsed_text, hit) { // parsed_text name should take precedence if available since it's the cleaner name property if (check.assigned(parsed_text) && check.assigned(parsed_text.name) && - hit.name.default.toLowerCase() === parsed_text.name.toLowerCase()) { + field.getStringValue(hit.name.default).toLowerCase() === parsed_text.name.toLowerCase()) { return 1; } // if no parsed_text check the text value as provided against result's default name - if (hit.name.default.toLowerCase() === text.toLowerCase()) { + if (field.getStringValue(hit.name.default).toLowerCase() === text.toLowerCase()) { return 1; } diff --git a/middleware/dedupe.js b/middleware/dedupe.js index a50b0494..0a3fd7e6 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -1,6 +1,7 @@ -var logger = require('pelias-logger').get('api'); -var _ = require('lodash'); -var isDifferent = require('../helper/diffPlaces').isDifferent; +const logger = require('pelias-logger').get('api'); +const _ = require('lodash'); +const isDifferent = require('../helper/diffPlaces').isDifferent; +const field = require('../helper/fieldValue'); function setup() { return dedupeResults; @@ -38,7 +39,7 @@ function dedupeResults(req, res, next) { logger.info('[dupe][replacing]', { query: req.clean.text, previous: uniqueResults[dupeIndex].source, - hit: hit.name.default + ' ' + hit.source + ':' + hit._id + hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id }); // replace previous dupe item with current hit uniqueResults[dupeIndex] = hit; @@ -48,7 +49,7 @@ function dedupeResults(req, res, next) { logger.info('[dupe][skipping]', { query: req.clean.text, previous: uniqueResults[dupeIndex].source, - hit: hit.name.default + ' ' + hit.source + ':' + hit._id + hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id }); } } diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 20c77f6b..491cc0ed 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -1,5 +1,6 @@ -var check = require('check-types'); -var _ = require('lodash'); +const check = require('check-types'); +const _ = require('lodash'); +const field = require('../helper/fieldValue'); var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL', 'CZE']; @@ -49,7 +50,7 @@ function flipNumberAndStreet(place) { flipped = ( place.address_parts.street + ' ' + place.address_parts.number ); // flip street name and housenumber - if( place.name.default === standard ){ + if( field.getStringValue(place.name.default) === standard ){ place.name.default = flipped; } } diff --git a/test/unit/helper/diffPlaces.js b/test/unit/helper/diffPlaces.js index a7dd692d..6fc27c36 100644 --- a/test/unit/helper/diffPlaces.js +++ b/test/unit/helper/diffPlaces.js @@ -166,6 +166,22 @@ module.exports.tests.dedupe = function(test, common) { t.false(isDifferent(item1, item2), 'should be the same'); t.end(); }); + + test('works with name aliases', function(t) { + var item1 = { + 'name': { + 'default': ['a','b'] // note the array + } + }; + var item2 = { + 'name': { + 'default': 'a' + } + }; + + t.false(isDifferent(item1, item2), 'should be the same'); + t.end(); + }); }; module.exports.all = function (tape, common) { diff --git a/test/unit/helper/fieldValue.js b/test/unit/helper/fieldValue.js new file mode 100644 index 00000000..d4784b9f --- /dev/null +++ b/test/unit/helper/fieldValue.js @@ -0,0 +1,41 @@ +const field = require('../../../helper/fieldValue'); + +module.exports.tests = {}; + +module.exports.tests.getStringValue = function(test) { + test('getStringValue', function(t) { + t.equal(field.getStringValue([]), '', 'empty array'); + t.equal(field.getStringValue(''), '', 'empty string'); + t.equal(field.getStringValue('foo'), 'foo', 'string'); + t.equal(field.getStringValue(['foo','bar']), 'foo', 'array'); + t.equal(field.getStringValue(['']), '', 'array with empty string'); + t.equal(field.getStringValue(-0), '-0', 'number'); + t.equal(field.getStringValue(+0), '0', 'number'); + t.equal(field.getStringValue({foo: 'bar'}), '[object Object]', '_.toString'); + t.end(); + }); +}; + +module.exports.tests.getArrayValue = function(test) { + test('getArrayValue', function(t) { + t.deepEqual(field.getArrayValue([]), [], 'empty array'); + t.deepEqual(field.getArrayValue(''), [], 'empty string'); + t.deepEqual(field.getArrayValue('foo'), ['foo'], 'string'); + t.deepEqual(field.getArrayValue(['foo','bar']), ['foo','bar'], 'array'); + t.deepEqual(field.getArrayValue(['']), [''], 'array with empty string'); + t.deepEqual(field.getArrayValue(-0), [0], 'number'); + t.deepEqual(field.getArrayValue(+0), [0], 'number'); + t.deepEqual(field.getArrayValue({foo: 'bar'}), [{foo: 'bar'}], '[*]'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('fieldValue: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 72ff6392..f5176752 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -638,6 +638,51 @@ module.exports.tests.non_optimal_conditions = (test, common) => { }; +// ensure that if elasticsearch returns an array of values for name.default +// .. that we handle this case and select the first element for the label. +module.exports.tests.nameAliases = function(test, common) { + test('name aliases', function(t) { + var aliases = [{ + '_type': 'example', + '_id': '1', + 'source': 'example', + 'layer': 'example', + 'name': { + 'default': ['Example1', 'Example2'] // note the array + }, + 'center_point': { + 'lon': 0, + 'lat': 0 + } + }]; + + const expected = { + type: 'FeatureCollection', + features: [{ + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [ 0, 0 ] + }, + properties: { + id: '1', + gid: 'example:example:1', + layer: 'example', + source: 'example', + source_id: undefined, + name: 'Example1' + } + }], + bbox: [ 0, 0, 0, 0 ] + }; + + var actual = geojsonify( {}, aliases ); + t.deepEquals(actual, expected); + t.end(); + }); + +}; + module.exports.all = (tape, common) => { function test(name, testFunction) { return tape(`geojsonify: ${name}`, testFunction); diff --git a/test/unit/helper/geojsonify_place_details.js b/test/unit/helper/geojsonify_place_details.js index c7f2c8ac..bb5bfa58 100644 --- a/test/unit/helper/geojsonify_place_details.js +++ b/test/unit/helper/geojsonify_place_details.js @@ -108,7 +108,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { }); test('\'empty\' string-type values should be output as \'\'', t => { - [ [], {}, '', 17, true, null, undefined ].forEach(empty_value => { + [ [], {}, '', true, null, undefined ].forEach(empty_value => { const source = { housenumber: empty_value, street: empty_value, diff --git a/test/unit/middleware/confidenceScore.js b/test/unit/middleware/confidenceScore.js index 9e10775f..316b2de6 100644 --- a/test/unit/middleware/confidenceScore.js +++ b/test/unit/middleware/confidenceScore.js @@ -201,6 +201,38 @@ module.exports.tests.confidenceScore = function(test, common) { t.equal(res.data[0].confidence, 0.28, 'score was set'); t.end(); }); + + test('works with name aliases', function(t) { + var req = { + clean: { + text: 'example', + parsed_text: { + number: 123, + street: 'example', + state: 'EG' + } + } + }; + var res = { + data: [{ + _score: 10, + found: true, + value: 1, + center_point: { lat: 100.1, lon: -50.5 }, + name: { default: ['test name1', 'test name2'] }, // note the array + }], + meta: { + scores: [10], + query_type: 'original' + } + }; + + t.doesNotThrow(() => { + confidenceScore(req, res, () => {}); + }); + t.equal(res.data[0].confidence, 0.28, 'score was set'); + t.end(); + }); }; module.exports.all = function (tape, common) { diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js index ca8a5cae..9b4d04c3 100644 --- a/test/unit/middleware/dedupe.js +++ b/test/unit/middleware/dedupe.js @@ -223,41 +223,71 @@ module.exports.tests.trump = function(test, common) { }); }); -test('osm with zip trumps openaddresses without zip', function (t) { - var req = { - clean: { - text: '100 Main St', - size: 100 - } - }; - var res = { - data: [ - { - 'name': { 'default': '100 Main St' }, - 'source': 'openaddresses', - 'source_id': '123456', - 'layer': 'address', - 'address_parts': {} - }, - { - 'name': { 'default': '100 Main St' }, - 'source': 'openstreetmap', - 'source_id': '654321', - 'layer': 'address', - 'address_parts': { - 'zip': '54321' + test('osm with zip trumps openaddresses without zip', function (t) { + var req = { + clean: { + text: '100 Main St', + size: 100 + } + }; + var res = { + data: [ + { + 'name': { 'default': '100 Main St' }, + 'source': 'openaddresses', + 'source_id': '123456', + 'layer': 'address', + 'address_parts': {} + }, + { + 'name': { 'default': '100 Main St' }, + 'source': 'openstreetmap', + 'source_id': '654321', + 'layer': 'address', + 'address_parts': { + 'zip': '54321' + } } + ] + }; + + var expectedCount = 1; + dedupe(req, res, function () { + t.equal(res.data.length, expectedCount, 'results have fewer items than before'); + t.deepEqual(res.data[0].source_id, '654321', 'openstreetmap result with zip won'); + t.end(); + }); + }); + + test('works with name aliases', function (t) { + var req = { + clean: { + text: '100 Main St', + size: 100 } - ] - }; + }; + var res = { + data: [ + { + 'name': { 'default': ['100 Main St'] }, // note the array + 'source': 'openaddresses', + 'source_id': '123456' + }, + { + 'name': { 'default': '100 Main St' }, + 'source': 'openstreetmap', + 'source_id': '654321' + } + ] + }; + + t.doesNotThrow(() => { + dedupe(req, res, () => {}); + }); - var expectedCount = 1; - dedupe(req, res, function () { - t.equal(res.data.length, expectedCount, 'results have fewer items than before'); - t.deepEqual(res.data[0].source_id, '654321', 'openstreetmap result with zip won'); + t.equal(res.data.length, 1, 'results have fewer items than before'); t.end(); }); -}); }; module.exports.all = function (tape, common) { diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index 1d4102f0..96fdc1d8 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -1,5 +1,5 @@ -var proxyquire = require('proxyquire'); +const proxyquire = require('proxyquire'); var customConfig = { generate: function generate() { @@ -40,7 +40,7 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { var deAddress = { '_id': 'test2', '_type': 'test', - 'name': { 'default': '23 Grolmanstraße' }, + 'name': { 'default': ['23 Grolmanstraße'] }, 'center_point': { 'lon': 13.321487, 'lat': 52.506781 }, 'address_parts': { 'zip': '10623', diff --git a/test/unit/run.js b/test/unit/run.js index 1a557733..52b6e519 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -32,6 +32,7 @@ var tests = [ require('./controller/predicates/is_request_sources_only_whosonfirst'), require('./helper/debug'), require('./helper/diffPlaces'), + require('./helper/fieldValue'), require('./helper/geojsonify_place_details'), require('./helper/geojsonify'), require('./helper/logging'), From e01fdb85c9a36e227a7b66771caf275cf6db116c Mon Sep 17 00:00:00 2001 From: missinglink Date: Thu, 22 Mar 2018 14:40:38 +0100 Subject: [PATCH 35/61] support aliases for label generation --- middleware/assignLabels.js | 2 +- package.json | 2 +- test/unit/middleware/assignLabels.js | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/middleware/assignLabels.js b/middleware/assignLabels.js index 1f072ddb..9ea6d74f 100644 --- a/middleware/assignLabels.js +++ b/middleware/assignLabels.js @@ -1,4 +1,4 @@ -var defaultLabelGenerator = require('pelias-labels'); +const defaultLabelGenerator = require('pelias-labels'); function setup(labelGenerator) { function middleware(req, res, next) { diff --git a/package.json b/package.json index e94c667b..7be0cb3e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "morgan": "^1.8.2", "pelias-categories": "1.2.0", "pelias-config": "2.14.0", - "pelias-labels": "1.7.0", + "pelias-labels": "1.8.0", "pelias-logger": "0.3.1", "pelias-microservice-wrapper": "1.3.0", "pelias-model": "5.3.2", diff --git a/test/unit/middleware/assignLabels.js b/test/unit/middleware/assignLabels.js index 26e32718..ebd7c22a 100644 --- a/test/unit/middleware/assignLabels.js +++ b/test/unit/middleware/assignLabels.js @@ -104,6 +104,33 @@ module.exports.tests.serialization = function(test, common) { }); + test('support name aliases', function(t) { + var assignLabels = require('../../../middleware/assignLabels')(); + + var res = { + data: [{ + name: { + default: ['name1','name2'] + } + }] + }; + + var expected = { + data: [{ + name: { + default: ['name1','name2'] + }, + label: 'name1' + }] + }; + + assignLabels({}, res, function () { + t.deepEqual(res, expected); + t.end(); + }); + + }); + }; module.exports.all = function (tape, common) { From 59f0f4d70ea36dac86bb1c40abaf587eabc541be Mon Sep 17 00:00:00 2001 From: missinglink Date: Mon, 26 Mar 2018 19:57:13 +0200 Subject: [PATCH 36/61] add line comment, fix line comment --- helper/fieldValue.js | 2 +- test/unit/helper/fieldValue.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/helper/fieldValue.js b/helper/fieldValue.js index 147bf625..6a9c5c10 100644 --- a/helper/fieldValue.js +++ b/helper/fieldValue.js @@ -17,7 +17,7 @@ function getStringValue(property) { return property; } - // array value, take first item in array (at this time only used for admin values) + // array value, take first item in array (at this time only used for admin & name values) if (_.isArray(property)) { return property[0]; } diff --git a/test/unit/helper/fieldValue.js b/test/unit/helper/fieldValue.js index d4784b9f..d3d007fe 100644 --- a/test/unit/helper/fieldValue.js +++ b/test/unit/helper/fieldValue.js @@ -11,6 +11,9 @@ module.exports.tests.getStringValue = function(test) { t.equal(field.getStringValue(['']), '', 'array with empty string'); t.equal(field.getStringValue(-0), '-0', 'number'); t.equal(field.getStringValue(+0), '0', 'number'); + + // note: this behaviour is not desirable, it was inherited during a refactor + // see: https://github.com/pelias/api/pull/1102 t.equal(field.getStringValue({foo: 'bar'}), '[object Object]', '_.toString'); t.end(); }); From a488cca83481b938cb2c621395755e2df2fad7b3 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 26 Mar 2018 10:36:01 -0400 Subject: [PATCH 37/61] Update Pelias attribution URL Connects https://github.com/pelias/pelias/issues/703 --- public/attribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/attribution.md b/public/attribution.md index 009450f6..f2b34bd3 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -1,5 +1,5 @@ ## Attribution -* Geocoding by [Pelias](https://mapzen.com/pelias) from [Mapzen](https://mapzen.com) +* Geocoding by [Pelias](https://pelias.io). * Data from * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation From 8bccfc4fd2a5a25b2286c368cd776f15141e5043 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 26 Mar 2018 10:39:16 -0400 Subject: [PATCH 38/61] Remove other instances of Mapzen --- middleware/headers.js | 5 ++--- package.json | 2 +- test/ciao/index.coffee | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/middleware/headers.js b/middleware/headers.js index a747be33..52bdffc9 100644 --- a/middleware/headers.js +++ b/middleware/headers.js @@ -1,12 +1,11 @@ - var pkg = require('../package'); function middleware(req, res, next){ res.header('Charset','utf8'); res.header('Cache-Control','public'); res.header('Server', 'Pelias/'+pkg.version); - res.header('X-Powered-By', 'mapzen'); + res.header('X-Powered-By', 'pelias'); next(); } -module.exports = middleware; \ No newline at end of file +module.exports = middleware; diff --git a/package.json b/package.json index 7be0cb3e..aa7939b1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "pelias-api", "version": "0.0.0-semantic-release", - "author": "mapzen", + "author": "pelias", "description": "Pelias API", "homepage": "https://github.com/pelias/api", "license": "MIT", diff --git a/test/ciao/index.coffee b/test/ciao/index.coffee index aac96fd9..add01625 100644 --- a/test/ciao/index.coffee +++ b/test/ciao/index.coffee @@ -19,4 +19,4 @@ response.should.have.header 'Server' response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/ #? vanity header correctly set -response.should.have.header 'X-Powered-By','mapzen' +response.should.have.header 'X-Powered-By','pelias' From db8194b0de5c645061b2e58cc4d9d0f0166c60e2 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 25 Mar 2018 11:21:39 -0400 Subject: [PATCH 39/61] Increase Node.js requirement to 6.0.0 or higher --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aa7939b1..20e4d849 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "url": "https://github.com/pelias/api/issues" }, "engines": { - "node": ">=4.0.0" + "node": ">=6.0.0" }, "dependencies": { "addressit": "1.5.0", From a4d93b357c59f46460455ca8acb67be944c8cda6 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 25 Mar 2018 11:25:37 -0400 Subject: [PATCH 40/61] Test on Node.js 6 and 8 in TravisCI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3ce77f80..bef32c83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ language: node_js notifications: email: false node_js: - - 4 - 6 + - 8 matrix: fast_finish: true env: From f437600ca0d76e4288039124c38d7b64dfb22a8e Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 25 Mar 2018 11:26:43 -0400 Subject: [PATCH 41/61] Remove TravisCI settings not needed with Node.js 8 --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index bef32c83..f91c3dfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,17 +7,13 @@ node_js: - 8 matrix: fast_finish: true -env: - global: - - BUILD_LEADER_ID=2 script: npm run travis before_install: - - npm i -g npm@^3.0.0 + - npm i -g npm before_script: - npm prune after_success: - - npm install -g npx - - npx -p node@8 npm run semantic-release + - npm run semantic-release branches: except: - /^v\d+\.\d+\.\d+$/ From 02aac0c13ee532beea30c616b1999bc0c22e4138 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 26 Mar 2018 14:52:42 -0400 Subject: [PATCH 42/61] Remove use stricts --- controller/place.js | 2 -- controller/search.js | 2 -- helper/debug.js | 1 - helper/fieldValue.js | 2 -- helper/geojsonify_place_details.js | 2 -- helper/stackTraceLine.js | 2 -- middleware/access_log.js | 2 -- middleware/confidenceScoreFallback.js | 2 -- middleware/normalizeParentIds.js | 2 -- middleware/renamePlacenames.js | 2 -- query/autocomplete.js | 2 -- query/reverse.js | 2 -- query/search.js | 1 - query/search_original.js | 1 - sanitizer/sanitizeAll.js | 1 - schema.js | 1 - service/configurations/Interpolation.js | 1 - service/configurations/Language.js | 1 - service/configurations/Libpostal.js | 1 - service/configurations/PlaceHolder.js | 1 - service/configurations/PointInPolygon.js | 1 - test/unit/app.js | 1 - test/unit/controller/coarse_reverse.js | 1 - test/unit/controller/libpostal.js | 1 - test/unit/controller/place.js | 1 - test/unit/controller/placeholder.js | 1 - test/unit/controller/predicates/has_parsed_text_properties.js | 1 - test/unit/controller/predicates/has_request_errors.js | 1 - test/unit/controller/predicates/has_request_parameter.js | 1 - test/unit/controller/predicates/has_response_data.js | 1 - test/unit/controller/predicates/has_results_at_layers.js | 1 - test/unit/controller/predicates/is_addressit_parse.js | 1 - test/unit/controller/predicates/is_admin_only_analysis.js | 1 - test/unit/controller/predicates/is_coarse_reverse.js | 1 - test/unit/controller/predicates/is_only_non_admin_layers.js | 1 - .../predicates/is_request_sources_only_whosonfirst.js | 1 - test/unit/controller/search.js | 1 - test/unit/controller/search_with_ids.js | 1 - test/unit/controller/structured_libpostal.js | 1 - test/unit/middleware/changeLanguage.js | 1 - test/unit/middleware/interpolate.js | 1 - test/unit/query/MockQuery.js | 1 - test/unit/sanitizer/_geo_reverse.js | 1 - test/unit/schema.js | 1 - 44 files changed, 55 deletions(-) diff --git a/controller/place.js b/controller/place.js index 51ce58d9..ac851359 100644 --- a/controller/place.js +++ b/controller/place.js @@ -1,5 +1,3 @@ -'use strict'; - const _ = require('lodash'); const retry = require('retry'); diff --git a/controller/search.js b/controller/search.js index 721c0dde..cc04f19e 100644 --- a/controller/search.js +++ b/controller/search.js @@ -1,5 +1,3 @@ -'use strict'; - const _ = require('lodash'); const searchService = require('../service/search'); diff --git a/helper/debug.js b/helper/debug.js index cf8b833c..79279c66 100644 --- a/helper/debug.js +++ b/helper/debug.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); class Debug { diff --git a/helper/fieldValue.js b/helper/fieldValue.js index 6a9c5c10..11ddef2d 100644 --- a/helper/fieldValue.js +++ b/helper/fieldValue.js @@ -1,5 +1,3 @@ -'use strict'; - const _ = require('lodash'); function getStringValue(property) { diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js index 4c2b60b7..91e12d73 100644 --- a/helper/geojsonify_place_details.js +++ b/helper/geojsonify_place_details.js @@ -1,5 +1,3 @@ -'use strict'; - const _ = require('lodash'); const field = require('./fieldValue'); diff --git a/helper/stackTraceLine.js b/helper/stackTraceLine.js index 67e56de2..e1312ae6 100644 --- a/helper/stackTraceLine.js +++ b/helper/stackTraceLine.js @@ -1,5 +1,3 @@ -'use strict'; - module.exports = () => { const stack = new Error().stack.split('\n'); let targetLine; diff --git a/middleware/access_log.js b/middleware/access_log.js index d87a2e32..e6df58d0 100644 --- a/middleware/access_log.js +++ b/middleware/access_log.js @@ -2,8 +2,6 @@ * Create a middleware that prints access logs via pelias-logger. */ -'use strict'; - var url = require( 'url' ); var _ = require( 'lodash' ); diff --git a/middleware/confidenceScoreFallback.js b/middleware/confidenceScoreFallback.js index 488bd721..96d378b8 100644 --- a/middleware/confidenceScoreFallback.js +++ b/middleware/confidenceScoreFallback.js @@ -1,5 +1,3 @@ -'use strict'; - /** * * Basic confidence score should be computed and returned for each item in the results. diff --git a/middleware/normalizeParentIds.js b/middleware/normalizeParentIds.js index 67a4874a..b5e02869 100644 --- a/middleware/normalizeParentIds.js +++ b/middleware/normalizeParentIds.js @@ -1,5 +1,3 @@ -'use strict'; - const logger = require('pelias-logger').get('api'); const Document = require('pelias-model').Document; const placeTypes = require('../helper/placeTypes'); diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 9f63cb6e..d1f36d73 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -1,5 +1,3 @@ -'use strict'; - const _ = require('lodash'); const PARENT_PROPS = require('../helper/placeTypes'); diff --git a/query/autocomplete.js b/query/autocomplete.js index e7dbed3f..8d4cf257 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -1,5 +1,3 @@ -'use strict'; - const peliasQuery = require('pelias-query'); const defaults = require('./autocomplete_defaults'); const textParser = require('./text_parser_addressit'); diff --git a/query/reverse.js b/query/reverse.js index a9c1d25c..1e928344 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,5 +1,3 @@ -'use strict'; - const peliasQuery = require('pelias-query'); const defaults = require('./reverse_defaults'); const check = require('check-types'); diff --git a/query/search.js b/query/search.js index 991e239c..12837496 100644 --- a/query/search.js +++ b/query/search.js @@ -1,4 +1,3 @@ -'use strict'; const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); diff --git a/query/search_original.js b/query/search_original.js index a6b666d6..c7b05248 100644 --- a/query/search_original.js +++ b/query/search_original.js @@ -1,4 +1,3 @@ -'use strict'; const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); diff --git a/sanitizer/sanitizeAll.js b/sanitizer/sanitizeAll.js index af6e1510..113c3116 100644 --- a/sanitizer/sanitizeAll.js +++ b/sanitizer/sanitizeAll.js @@ -1,4 +1,3 @@ -'use strict'; function sanitize( req, sanitizers ){ // init an object to store clean (sanitized) input parameters if not initialized req.clean = req.clean || {}; diff --git a/schema.js b/schema.js index 440da818..00c31a2e 100644 --- a/schema.js +++ b/schema.js @@ -1,4 +1,3 @@ -'use strict'; const Joi = require('joi'); diff --git a/service/configurations/Interpolation.js b/service/configurations/Interpolation.js index 1bb08d77..9995e053 100644 --- a/service/configurations/Interpolation.js +++ b/service/configurations/Interpolation.js @@ -1,4 +1,3 @@ -'use strict'; const url = require('url'); diff --git a/service/configurations/Language.js b/service/configurations/Language.js index ef528d38..97096a87 100644 --- a/service/configurations/Language.js +++ b/service/configurations/Language.js @@ -1,4 +1,3 @@ -'use strict'; const url = require('url'); diff --git a/service/configurations/Libpostal.js b/service/configurations/Libpostal.js index 9cf1de2b..997a8a17 100644 --- a/service/configurations/Libpostal.js +++ b/service/configurations/Libpostal.js @@ -1,4 +1,3 @@ -'use strict'; const url = require('url'); diff --git a/service/configurations/PlaceHolder.js b/service/configurations/PlaceHolder.js index c25f8abc..a0ad6ee9 100644 --- a/service/configurations/PlaceHolder.js +++ b/service/configurations/PlaceHolder.js @@ -1,4 +1,3 @@ -'use strict'; const url = require('url'); diff --git a/service/configurations/PointInPolygon.js b/service/configurations/PointInPolygon.js index 58b52093..989b854a 100644 --- a/service/configurations/PointInPolygon.js +++ b/service/configurations/PointInPolygon.js @@ -1,4 +1,3 @@ -'use strict'; const url = require('url'); diff --git a/test/unit/app.js b/test/unit/app.js index 479e2af0..192e9034 100644 --- a/test/unit/app.js +++ b/test/unit/app.js @@ -1,4 +1,3 @@ -'use strict'; const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/coarse_reverse.js b/test/unit/controller/coarse_reverse.js index df231075..ce6bb886 100644 --- a/test/unit/controller/coarse_reverse.js +++ b/test/unit/controller/coarse_reverse.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../controller/coarse_reverse'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/libpostal.js b/test/unit/controller/libpostal.js index cf62fff5..708130c4 100644 --- a/test/unit/controller/libpostal.js +++ b/test/unit/controller/libpostal.js @@ -1,4 +1,3 @@ -'use strict'; const proxyquire = require('proxyquire').noCallThru(); const libpostal = require('../../../controller/libpostal'); diff --git a/test/unit/controller/place.js b/test/unit/controller/place.js index b76d69c9..8bbb5b00 100644 --- a/test/unit/controller/place.js +++ b/test/unit/controller/place.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../controller/place'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/placeholder.js b/test/unit/controller/placeholder.js index efb0fc2a..ede7982d 100644 --- a/test/unit/controller/placeholder.js +++ b/test/unit/controller/placeholder.js @@ -1,4 +1,3 @@ -'use strict'; const placeholder = require('../../../controller/placeholder'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/predicates/has_parsed_text_properties.js b/test/unit/controller/predicates/has_parsed_text_properties.js index 9307d46d..3ee955d7 100644 --- a/test/unit/controller/predicates/has_parsed_text_properties.js +++ b/test/unit/controller/predicates/has_parsed_text_properties.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const has_parsed_text_properties = require('../../../../controller/predicates/has_parsed_text_properties'); diff --git a/test/unit/controller/predicates/has_request_errors.js b/test/unit/controller/predicates/has_request_errors.js index bd1868aa..53a3925f 100644 --- a/test/unit/controller/predicates/has_request_errors.js +++ b/test/unit/controller/predicates/has_request_errors.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const has_request_errors = require('../../../../controller/predicates/has_request_errors'); diff --git a/test/unit/controller/predicates/has_request_parameter.js b/test/unit/controller/predicates/has_request_parameter.js index 099ba39d..9040fc07 100644 --- a/test/unit/controller/predicates/has_request_parameter.js +++ b/test/unit/controller/predicates/has_request_parameter.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const has_request_parameter = require('../../../../controller/predicates/has_request_parameter'); diff --git a/test/unit/controller/predicates/has_response_data.js b/test/unit/controller/predicates/has_response_data.js index f073fc4d..265e122b 100644 --- a/test/unit/controller/predicates/has_response_data.js +++ b/test/unit/controller/predicates/has_response_data.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const has_response_data = require('../../../../controller/predicates/has_response_data'); diff --git a/test/unit/controller/predicates/has_results_at_layers.js b/test/unit/controller/predicates/has_results_at_layers.js index f81e9045..d1984580 100644 --- a/test/unit/controller/predicates/has_results_at_layers.js +++ b/test/unit/controller/predicates/has_results_at_layers.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const has_results_at_layers = require('../../../../controller/predicates/has_results_at_layers'); diff --git a/test/unit/controller/predicates/is_addressit_parse.js b/test/unit/controller/predicates/is_addressit_parse.js index d11734ba..4d64e296 100644 --- a/test/unit/controller/predicates/is_addressit_parse.js +++ b/test/unit/controller/predicates/is_addressit_parse.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const is_addressit_parse = require('../../../../controller/predicates/is_addressit_parse'); diff --git a/test/unit/controller/predicates/is_admin_only_analysis.js b/test/unit/controller/predicates/is_admin_only_analysis.js index 4e90bf34..e6b5e323 100644 --- a/test/unit/controller/predicates/is_admin_only_analysis.js +++ b/test/unit/controller/predicates/is_admin_only_analysis.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const is_admin_only_analysis = require('../../../../controller/predicates/is_admin_only_analysis'); diff --git a/test/unit/controller/predicates/is_coarse_reverse.js b/test/unit/controller/predicates/is_coarse_reverse.js index 33b0611d..baeecd7c 100644 --- a/test/unit/controller/predicates/is_coarse_reverse.js +++ b/test/unit/controller/predicates/is_coarse_reverse.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const is_coarse_reverse = require('../../../../controller/predicates/is_coarse_reverse'); diff --git a/test/unit/controller/predicates/is_only_non_admin_layers.js b/test/unit/controller/predicates/is_only_non_admin_layers.js index 51e1b796..17d7f428 100644 --- a/test/unit/controller/predicates/is_only_non_admin_layers.js +++ b/test/unit/controller/predicates/is_only_non_admin_layers.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const is_only_non_admin_layers = require('../../../../controller/predicates/is_only_non_admin_layers'); diff --git a/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js b/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js index 24f861ad..b8fecd06 100644 --- a/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js +++ b/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js @@ -1,4 +1,3 @@ -'use strict'; const _ = require('lodash'); const is_request_sources_only_whosonfirst = require('../../../../controller/predicates/is_request_sources_only_whosonfirst'); diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js index 3f80f7d1..078a4ece 100644 --- a/test/unit/controller/search.js +++ b/test/unit/controller/search.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../controller/search'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/search_with_ids.js b/test/unit/controller/search_with_ids.js index ff884be9..0723539d 100644 --- a/test/unit/controller/search_with_ids.js +++ b/test/unit/controller/search_with_ids.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../controller/search_with_ids'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/structured_libpostal.js b/test/unit/controller/structured_libpostal.js index eb4aea12..d39d0532 100644 --- a/test/unit/controller/structured_libpostal.js +++ b/test/unit/controller/structured_libpostal.js @@ -1,4 +1,3 @@ -'use strict'; const proxyquire = require('proxyquire').noCallThru(); const libpostal = require('../../../controller/structured_libpostal'); diff --git a/test/unit/middleware/changeLanguage.js b/test/unit/middleware/changeLanguage.js index cc0688d9..7536810c 100644 --- a/test/unit/middleware/changeLanguage.js +++ b/test/unit/middleware/changeLanguage.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../middleware/changeLanguage'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/middleware/interpolate.js b/test/unit/middleware/interpolate.js index 6dbf1ae6..d90210ef 100644 --- a/test/unit/middleware/interpolate.js +++ b/test/unit/middleware/interpolate.js @@ -1,4 +1,3 @@ -'use strict'; const setup = require('../../../middleware/interpolate'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/query/MockQuery.js b/test/unit/query/MockQuery.js index e7202cc9..2cbcf78d 100644 --- a/test/unit/query/MockQuery.js +++ b/test/unit/query/MockQuery.js @@ -1,4 +1,3 @@ -'use strict'; module.exports = class MockQuery { constructor() { diff --git a/test/unit/sanitizer/_geo_reverse.js b/test/unit/sanitizer/_geo_reverse.js index c2fa6bd9..0afc2b15 100644 --- a/test/unit/sanitizer/_geo_reverse.js +++ b/test/unit/sanitizer/_geo_reverse.js @@ -1,4 +1,3 @@ -'use strict'; const sanitizer = require('../../../sanitizer/_geo_reverse')(); const defaults = require('../../../query/reverse_defaults'); diff --git a/test/unit/schema.js b/test/unit/schema.js index 1278ee59..93e93964 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -1,4 +1,3 @@ -'use strict'; const Joi = require('joi'); const schema = require('../../schema'); From 204f5297de3344a8e3e4b58eb7be869def5a1e26 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 26 Mar 2018 14:59:53 -0400 Subject: [PATCH 43/61] Remove all leading newlines in files --- controller/markdownToHtml.js | 1 - helper/geojsonify.js | 1 - middleware/404.js | 1 - middleware/changeLanguage.js | 1 - middleware/cors.js | 1 - middleware/jsonp.js | 1 - middleware/options.js | 1 - middleware/requestLanguage.js | 1 - query/autocomplete_defaults.js | 1 - query/reverse_defaults.js | 1 - query/search.js | 1 - query/search_defaults.js | 1 - query/search_original.js | 1 - query/text_parser_addressit.js | 1 - query/view/boost_exact_matches.js | 1 - query/view/focus_selected_layers.js | 1 - query/view/ngrams_last_token_only.js | 1 - query/view/ngrams_strict.js | 1 - query/view/phrase_first_tokens_only.js | 1 - query/view/pop_subquery.js | 1 - sanitizer/_single_scalar_parameters.js | 1 - sanitizer/_tokenizer.js | 1 - sanitizer/place.js | 1 - sanitizer/reverse.js | 1 - sanitizer/wrap.js | 1 - service/configurations/Interpolation.js | 1 - service/configurations/Language.js | 1 - service/configurations/Libpostal.js | 1 - service/configurations/PlaceHolder.js | 1 - service/configurations/PointInPolygon.js | 1 - service/mget.js | 1 - service/search.js | 1 - test/ciao_test_data.js | 1 - test/unit/app.js | 1 - test/unit/controller/coarse_reverse.js | 1 - test/unit/controller/index.js | 1 - test/unit/controller/libpostal.js | 1 - test/unit/controller/place.js | 1 - test/unit/controller/placeholder.js | 1 - test/unit/controller/predicates/has_parsed_text_properties.js | 1 - test/unit/controller/predicates/has_request_errors.js | 1 - test/unit/controller/predicates/has_request_parameter.js | 1 - test/unit/controller/predicates/has_response_data.js | 1 - test/unit/controller/predicates/has_results_at_layers.js | 1 - test/unit/controller/predicates/is_addressit_parse.js | 1 - test/unit/controller/predicates/is_admin_only_analysis.js | 1 - test/unit/controller/predicates/is_coarse_reverse.js | 1 - test/unit/controller/predicates/is_only_non_admin_layers.js | 1 - .../controller/predicates/is_request_sources_only_whosonfirst.js | 1 - test/unit/controller/search.js | 1 - test/unit/controller/search_with_ids.js | 1 - test/unit/controller/structured_libpostal.js | 1 - test/unit/fixture/autocomplete_boundary_country.js | 1 - test/unit/fixture/autocomplete_linguistic_final_token.js | 1 - test/unit/fixture/autocomplete_linguistic_focus.js | 1 - test/unit/fixture/autocomplete_linguistic_focus_null_island.js | 1 - test/unit/fixture/autocomplete_linguistic_multiple_tokens.js | 1 - test/unit/fixture/autocomplete_linguistic_only.js | 1 - test/unit/fixture/autocomplete_linguistic_with_admin.js | 1 - test/unit/fixture/autocomplete_single_character_street.js | 1 - test/unit/fixture/autocomplete_with_layer_filtering.js | 1 - test/unit/fixture/autocomplete_with_source_filtering.js | 1 - test/unit/fixture/search_boundary_country_original.js | 1 - test/unit/fixture/search_linguistic_bbox_original.js | 1 - test/unit/fixture/search_linguistic_focus_bbox_original.js | 1 - .../unit/fixture/search_linguistic_focus_null_island_original.js | 1 - test/unit/fixture/search_linguistic_focus_original.js | 1 - test/unit/fixture/search_linguistic_only_original.js | 1 - test/unit/fixture/search_partial_address_original.js | 1 - test/unit/fixture/search_regions_address_original.js | 1 - test/unit/fixture/search_with_source_filtering_original.js | 1 - test/unit/middleware/changeLanguage.js | 1 - test/unit/middleware/interpolate.js | 1 - test/unit/middleware/localNamingConventions.js | 1 - test/unit/middleware/requestLanguage.js | 1 - test/unit/query/MockQuery.js | 1 - test/unit/query/autocomplete_defaults.js | 1 - test/unit/query/reverse_defaults.js | 1 - test/unit/query/search_defaults.js | 1 - test/unit/sanitizer/_geo_reverse.js | 1 - test/unit/sanitizer/wrap.js | 1 - test/unit/schema.js | 1 - 82 files changed, 82 deletions(-) diff --git a/controller/markdownToHtml.js b/controller/markdownToHtml.js index 656afe01..774127bb 100644 --- a/controller/markdownToHtml.js +++ b/controller/markdownToHtml.js @@ -1,4 +1,3 @@ - var markdown = require('markdown').markdown; var fs = require('fs'); diff --git a/helper/geojsonify.js b/helper/geojsonify.js index 5c571fa7..036fe773 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -1,4 +1,3 @@ - const GeoJSON = require('geojson'); const extent = require('@mapbox/geojson-extent'); const logger = require('pelias-logger').get('geojsonify'); diff --git a/middleware/404.js b/middleware/404.js index 65c25467..db4d013f 100644 --- a/middleware/404.js +++ b/middleware/404.js @@ -1,4 +1,3 @@ - // handle not found errors function middleware(req, res) { res.header('Cache-Control','public'); diff --git a/middleware/changeLanguage.js b/middleware/changeLanguage.js index 91bd5ba0..3d02a296 100644 --- a/middleware/changeLanguage.js +++ b/middleware/changeLanguage.js @@ -1,4 +1,3 @@ - var logger = require( 'pelias-logger' ).get( 'api' ); const _ = require('lodash'); diff --git a/middleware/cors.js b/middleware/cors.js index 6d30529f..51ee56e4 100644 --- a/middleware/cors.js +++ b/middleware/cors.js @@ -1,4 +1,3 @@ - function middleware(req, res, next){ res.header('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Methods', 'GET, OPTIONS'); diff --git a/middleware/jsonp.js b/middleware/jsonp.js index 1a54716e..3f06dbf6 100644 --- a/middleware/jsonp.js +++ b/middleware/jsonp.js @@ -1,4 +1,3 @@ - function middleware(req, res, next){ // store old json function diff --git a/middleware/options.js b/middleware/options.js index f6ca8767..7bb39093 100644 --- a/middleware/options.js +++ b/middleware/options.js @@ -1,4 +1,3 @@ - /** this functionality is required by CORS as the browser will send an HTTP OPTIONS request before performing the CORS request. diff --git a/middleware/requestLanguage.js b/middleware/requestLanguage.js index c7532fc9..d6e154ab 100644 --- a/middleware/requestLanguage.js +++ b/middleware/requestLanguage.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const logger = require( 'pelias-logger' ).get( 'api' ); diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 34f396ae..dbba5861 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); var _ = require('lodash'); diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index bce72057..cbcf7a65 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); var _ = require('lodash'); diff --git a/query/search.js b/query/search.js index 12837496..26ae1e40 100644 --- a/query/search.js +++ b/query/search.js @@ -1,4 +1,3 @@ - const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); const textParser = require('./text_parser'); diff --git a/query/search_defaults.js b/query/search_defaults.js index c0d2a6a0..92154ee8 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); var _ = require('lodash'); diff --git a/query/search_original.js b/query/search_original.js index c7b05248..6b9856b7 100644 --- a/query/search_original.js +++ b/query/search_original.js @@ -1,4 +1,3 @@ - const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); const textParser = require('./text_parser_addressit'); diff --git a/query/text_parser_addressit.js b/query/text_parser_addressit.js index 84979c15..8a3907a8 100644 --- a/query/text_parser_addressit.js +++ b/query/text_parser_addressit.js @@ -1,4 +1,3 @@ - var logger = require('pelias-logger').get('api'); var placeTypes = require('../helper/placeTypes'); diff --git a/query/view/boost_exact_matches.js b/query/view/boost_exact_matches.js index 9af56cfb..06155260 100644 --- a/query/view/boost_exact_matches.js +++ b/query/view/boost_exact_matches.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'), searchDefaults = require('../search_defaults'); diff --git a/query/view/focus_selected_layers.js b/query/view/focus_selected_layers.js index 9d23998c..4fda1780 100644 --- a/query/view/focus_selected_layers.js +++ b/query/view/focus_selected_layers.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); /** diff --git a/query/view/ngrams_last_token_only.js b/query/view/ngrams_last_token_only.js index 12be4584..db2862c3 100644 --- a/query/view/ngrams_last_token_only.js +++ b/query/view/ngrams_last_token_only.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'), ngrams_strict = require('./ngrams_strict'); diff --git a/query/view/ngrams_strict.js b/query/view/ngrams_strict.js index b0d7ae94..f5fd815e 100644 --- a/query/view/ngrams_strict.js +++ b/query/view/ngrams_strict.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); /** diff --git a/query/view/phrase_first_tokens_only.js b/query/view/phrase_first_tokens_only.js index 7ab4539b..795e4beb 100644 --- a/query/view/phrase_first_tokens_only.js +++ b/query/view/phrase_first_tokens_only.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'); /** diff --git a/query/view/pop_subquery.js b/query/view/pop_subquery.js index fc8f6804..4474fa4d 100644 --- a/query/view/pop_subquery.js +++ b/query/view/pop_subquery.js @@ -1,4 +1,3 @@ - var peliasQuery = require('pelias-query'), check = require('check-types'); diff --git a/sanitizer/_single_scalar_parameters.js b/sanitizer/_single_scalar_parameters.js index 4c49fc21..2363ef65 100644 --- a/sanitizer/_single_scalar_parameters.js +++ b/sanitizer/_single_scalar_parameters.js @@ -1,4 +1,3 @@ - var _ = require('lodash'), check = require('check-types'); diff --git a/sanitizer/_tokenizer.js b/sanitizer/_tokenizer.js index 18b5d3e0..081a9121 100644 --- a/sanitizer/_tokenizer.js +++ b/sanitizer/_tokenizer.js @@ -1,4 +1,3 @@ - var check = require('check-types'); /** diff --git a/sanitizer/place.js b/sanitizer/place.js index 4d158dc3..9b7c7fdb 100644 --- a/sanitizer/place.js +++ b/sanitizer/place.js @@ -1,4 +1,3 @@ - var sanitizeAll = require('../sanitizer/sanitizeAll'), sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), diff --git a/sanitizer/reverse.js b/sanitizer/reverse.js index 457e81a5..ce14e862 100644 --- a/sanitizer/reverse.js +++ b/sanitizer/reverse.js @@ -1,4 +1,3 @@ - var type_mapping = require('../helper/type_mapping'); var sanitizeAll = require('../sanitizer/sanitizeAll'), sanitizers = { diff --git a/sanitizer/wrap.js b/sanitizer/wrap.js index bc8a9cd2..bca70220 100644 --- a/sanitizer/wrap.js +++ b/sanitizer/wrap.js @@ -1,4 +1,3 @@ - /** normalize co-ordinates that lie outside of the normal ranges. diff --git a/service/configurations/Interpolation.js b/service/configurations/Interpolation.js index 9995e053..ff7c685b 100644 --- a/service/configurations/Interpolation.js +++ b/service/configurations/Interpolation.js @@ -1,4 +1,3 @@ - const url = require('url'); const _ = require('lodash'); diff --git a/service/configurations/Language.js b/service/configurations/Language.js index 97096a87..c20fa861 100644 --- a/service/configurations/Language.js +++ b/service/configurations/Language.js @@ -1,4 +1,3 @@ - const url = require('url'); const _ = require('lodash'); diff --git a/service/configurations/Libpostal.js b/service/configurations/Libpostal.js index 997a8a17..0a895846 100644 --- a/service/configurations/Libpostal.js +++ b/service/configurations/Libpostal.js @@ -1,4 +1,3 @@ - const url = require('url'); const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration; diff --git a/service/configurations/PlaceHolder.js b/service/configurations/PlaceHolder.js index a0ad6ee9..055863a9 100644 --- a/service/configurations/PlaceHolder.js +++ b/service/configurations/PlaceHolder.js @@ -1,4 +1,3 @@ - const url = require('url'); const _ = require('lodash'); diff --git a/service/configurations/PointInPolygon.js b/service/configurations/PointInPolygon.js index 989b854a..e13986fc 100644 --- a/service/configurations/PointInPolygon.js +++ b/service/configurations/PointInPolygon.js @@ -1,4 +1,3 @@ - const url = require('url'); const _ = require('lodash'); diff --git a/service/mget.js b/service/mget.js index b68a5bed..346ddb90 100644 --- a/service/mget.js +++ b/service/mget.js @@ -1,4 +1,3 @@ - /** query must be an array of hashes, structured like so: diff --git a/service/search.js b/service/search.js index dcdd7093..1db7cef6 100644 --- a/service/search.js +++ b/service/search.js @@ -1,4 +1,3 @@ - /** cmd can be any valid ES query command diff --git a/test/ciao_test_data.js b/test/ciao_test_data.js index 55872f67..91eaf11d 100644 --- a/test/ciao_test_data.js +++ b/test/ciao_test_data.js @@ -1,4 +1,3 @@ - /** Test data required by the ciao test suite. diff --git a/test/unit/app.js b/test/unit/app.js index 192e9034..ec786f2f 100644 --- a/test/unit/app.js +++ b/test/unit/app.js @@ -1,4 +1,3 @@ - const proxyquire = require('proxyquire').noCallThru(); module.exports.tests = {}; diff --git a/test/unit/controller/coarse_reverse.js b/test/unit/controller/coarse_reverse.js index ce6bb886..2a13d4a3 100644 --- a/test/unit/controller/coarse_reverse.js +++ b/test/unit/controller/coarse_reverse.js @@ -1,4 +1,3 @@ - const setup = require('../../../controller/coarse_reverse'); const proxyquire = require('proxyquire').noCallThru(); const _ = require('lodash'); diff --git a/test/unit/controller/index.js b/test/unit/controller/index.js index cffe1030..b6a5a396 100644 --- a/test/unit/controller/index.js +++ b/test/unit/controller/index.js @@ -1,4 +1,3 @@ - var setup = require('../../../controller/markdownToHtml'); module.exports.tests = {}; diff --git a/test/unit/controller/libpostal.js b/test/unit/controller/libpostal.js index 708130c4..56b47521 100644 --- a/test/unit/controller/libpostal.js +++ b/test/unit/controller/libpostal.js @@ -1,4 +1,3 @@ - const proxyquire = require('proxyquire').noCallThru(); const libpostal = require('../../../controller/libpostal'); const _ = require('lodash'); diff --git a/test/unit/controller/place.js b/test/unit/controller/place.js index 8bbb5b00..533db310 100644 --- a/test/unit/controller/place.js +++ b/test/unit/controller/place.js @@ -1,4 +1,3 @@ - const setup = require('../../../controller/place'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/placeholder.js b/test/unit/controller/placeholder.js index ede7982d..82ace9bd 100644 --- a/test/unit/controller/placeholder.js +++ b/test/unit/controller/placeholder.js @@ -1,4 +1,3 @@ - const placeholder = require('../../../controller/placeholder'); const proxyquire = require('proxyquire').noCallThru(); const mock_logger = require('pelias-mock-logger'); diff --git a/test/unit/controller/predicates/has_parsed_text_properties.js b/test/unit/controller/predicates/has_parsed_text_properties.js index 3ee955d7..b1635d94 100644 --- a/test/unit/controller/predicates/has_parsed_text_properties.js +++ b/test/unit/controller/predicates/has_parsed_text_properties.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const has_parsed_text_properties = require('../../../../controller/predicates/has_parsed_text_properties'); diff --git a/test/unit/controller/predicates/has_request_errors.js b/test/unit/controller/predicates/has_request_errors.js index 53a3925f..437ecaf7 100644 --- a/test/unit/controller/predicates/has_request_errors.js +++ b/test/unit/controller/predicates/has_request_errors.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const has_request_errors = require('../../../../controller/predicates/has_request_errors'); diff --git a/test/unit/controller/predicates/has_request_parameter.js b/test/unit/controller/predicates/has_request_parameter.js index 9040fc07..3a4e33b1 100644 --- a/test/unit/controller/predicates/has_request_parameter.js +++ b/test/unit/controller/predicates/has_request_parameter.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const has_request_parameter = require('../../../../controller/predicates/has_request_parameter'); diff --git a/test/unit/controller/predicates/has_response_data.js b/test/unit/controller/predicates/has_response_data.js index 265e122b..881a6a72 100644 --- a/test/unit/controller/predicates/has_response_data.js +++ b/test/unit/controller/predicates/has_response_data.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const has_response_data = require('../../../../controller/predicates/has_response_data'); diff --git a/test/unit/controller/predicates/has_results_at_layers.js b/test/unit/controller/predicates/has_results_at_layers.js index d1984580..608e2ff1 100644 --- a/test/unit/controller/predicates/has_results_at_layers.js +++ b/test/unit/controller/predicates/has_results_at_layers.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const has_results_at_layers = require('../../../../controller/predicates/has_results_at_layers'); diff --git a/test/unit/controller/predicates/is_addressit_parse.js b/test/unit/controller/predicates/is_addressit_parse.js index 4d64e296..219c36ff 100644 --- a/test/unit/controller/predicates/is_addressit_parse.js +++ b/test/unit/controller/predicates/is_addressit_parse.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const is_addressit_parse = require('../../../../controller/predicates/is_addressit_parse'); diff --git a/test/unit/controller/predicates/is_admin_only_analysis.js b/test/unit/controller/predicates/is_admin_only_analysis.js index e6b5e323..1f8232e1 100644 --- a/test/unit/controller/predicates/is_admin_only_analysis.js +++ b/test/unit/controller/predicates/is_admin_only_analysis.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const is_admin_only_analysis = require('../../../../controller/predicates/is_admin_only_analysis'); diff --git a/test/unit/controller/predicates/is_coarse_reverse.js b/test/unit/controller/predicates/is_coarse_reverse.js index baeecd7c..e92bb177 100644 --- a/test/unit/controller/predicates/is_coarse_reverse.js +++ b/test/unit/controller/predicates/is_coarse_reverse.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const is_coarse_reverse = require('../../../../controller/predicates/is_coarse_reverse'); diff --git a/test/unit/controller/predicates/is_only_non_admin_layers.js b/test/unit/controller/predicates/is_only_non_admin_layers.js index 17d7f428..eb0ba2b1 100644 --- a/test/unit/controller/predicates/is_only_non_admin_layers.js +++ b/test/unit/controller/predicates/is_only_non_admin_layers.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const is_only_non_admin_layers = require('../../../../controller/predicates/is_only_non_admin_layers'); diff --git a/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js b/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js index b8fecd06..c3686c47 100644 --- a/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js +++ b/test/unit/controller/predicates/is_request_sources_only_whosonfirst.js @@ -1,4 +1,3 @@ - const _ = require('lodash'); const is_request_sources_only_whosonfirst = require('../../../../controller/predicates/is_request_sources_only_whosonfirst'); diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js index 078a4ece..0e274d4e 100644 --- a/test/unit/controller/search.js +++ b/test/unit/controller/search.js @@ -1,4 +1,3 @@ - const setup = require('../../../controller/search'); const proxyquire = require('proxyquire').noCallThru(); diff --git a/test/unit/controller/search_with_ids.js b/test/unit/controller/search_with_ids.js index 0723539d..250e32f9 100644 --- a/test/unit/controller/search_with_ids.js +++ b/test/unit/controller/search_with_ids.js @@ -1,4 +1,3 @@ - const setup = require('../../../controller/search_with_ids'); const proxyquire = require('proxyquire').noCallThru(); const mocklogger = require('pelias-mock-logger'); diff --git a/test/unit/controller/structured_libpostal.js b/test/unit/controller/structured_libpostal.js index d39d0532..142d8260 100644 --- a/test/unit/controller/structured_libpostal.js +++ b/test/unit/controller/structured_libpostal.js @@ -1,4 +1,3 @@ - const proxyquire = require('proxyquire').noCallThru(); const libpostal = require('../../../controller/structured_libpostal'); const _ = require('lodash'); diff --git a/test/unit/fixture/autocomplete_boundary_country.js b/test/unit/fixture/autocomplete_boundary_country.js index f0ac8fb7..928c3efe 100644 --- a/test/unit/fixture/autocomplete_boundary_country.js +++ b/test/unit/fixture/autocomplete_boundary_country.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_final_token.js b/test/unit/fixture/autocomplete_linguistic_final_token.js index c27e64f5..4f250ec3 100644 --- a/test/unit/fixture/autocomplete_linguistic_final_token.js +++ b/test/unit/fixture/autocomplete_linguistic_final_token.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index d6aa0aa7..219bea36 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 8dd3f1e6..80e44f81 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js index 07a06d88..b07219ac 100644 --- a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js +++ b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_only.js b/test/unit/fixture/autocomplete_linguistic_only.js index 88a7d013..036bd180 100644 --- a/test/unit/fixture/autocomplete_linguistic_only.js +++ b/test/unit/fixture/autocomplete_linguistic_only.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_linguistic_with_admin.js b/test/unit/fixture/autocomplete_linguistic_with_admin.js index 2fcd25ca..12940e19 100644 --- a/test/unit/fixture/autocomplete_linguistic_with_admin.js +++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_single_character_street.js b/test/unit/fixture/autocomplete_single_character_street.js index 356ef884..fe4e1b5c 100644 --- a/test/unit/fixture/autocomplete_single_character_street.js +++ b/test/unit/fixture/autocomplete_single_character_street.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_with_layer_filtering.js b/test/unit/fixture/autocomplete_with_layer_filtering.js index 6f9d5e0f..ac5971c4 100644 --- a/test/unit/fixture/autocomplete_with_layer_filtering.js +++ b/test/unit/fixture/autocomplete_with_layer_filtering.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/autocomplete_with_source_filtering.js b/test/unit/fixture/autocomplete_with_source_filtering.js index cebd7eea..ada4c953 100644 --- a/test/unit/fixture/autocomplete_with_source_filtering.js +++ b/test/unit/fixture/autocomplete_with_source_filtering.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_boundary_country_original.js b/test/unit/fixture/search_boundary_country_original.js index 94f867b2..0ea31ab3 100644 --- a/test/unit/fixture/search_boundary_country_original.js +++ b/test/unit/fixture/search_boundary_country_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_linguistic_bbox_original.js b/test/unit/fixture/search_linguistic_bbox_original.js index b8dbf3a1..56fd90bf 100644 --- a/test/unit/fixture/search_linguistic_bbox_original.js +++ b/test/unit/fixture/search_linguistic_bbox_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_linguistic_focus_bbox_original.js b/test/unit/fixture/search_linguistic_focus_bbox_original.js index ebc5f701..43bea77d 100644 --- a/test/unit/fixture/search_linguistic_focus_bbox_original.js +++ b/test/unit/fixture/search_linguistic_focus_bbox_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_linguistic_focus_null_island_original.js b/test/unit/fixture/search_linguistic_focus_null_island_original.js index 8f6fe381..06b598cc 100644 --- a/test/unit/fixture/search_linguistic_focus_null_island_original.js +++ b/test/unit/fixture/search_linguistic_focus_null_island_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_linguistic_focus_original.js b/test/unit/fixture/search_linguistic_focus_original.js index 38273273..9ece3b86 100644 --- a/test/unit/fixture/search_linguistic_focus_original.js +++ b/test/unit/fixture/search_linguistic_focus_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_linguistic_only_original.js b/test/unit/fixture/search_linguistic_only_original.js index 490eb0c9..1029c7f9 100644 --- a/test/unit/fixture/search_linguistic_only_original.js +++ b/test/unit/fixture/search_linguistic_only_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/fixture/search_partial_address_original.js b/test/unit/fixture/search_partial_address_original.js index b0c82a3f..b0cced91 100644 --- a/test/unit/fixture/search_partial_address_original.js +++ b/test/unit/fixture/search_partial_address_original.js @@ -1,4 +1,3 @@ - var vs = require('../../../query/search_defaults'); module.exports = { diff --git a/test/unit/fixture/search_regions_address_original.js b/test/unit/fixture/search_regions_address_original.js index 400f561b..1d211b8d 100644 --- a/test/unit/fixture/search_regions_address_original.js +++ b/test/unit/fixture/search_regions_address_original.js @@ -1,4 +1,3 @@ - var vs = require('../../../query/search_defaults'); module.exports = { diff --git a/test/unit/fixture/search_with_source_filtering_original.js b/test/unit/fixture/search_with_source_filtering_original.js index 24da9468..650fad61 100644 --- a/test/unit/fixture/search_with_source_filtering_original.js +++ b/test/unit/fixture/search_with_source_filtering_original.js @@ -1,4 +1,3 @@ - module.exports = { 'query': { 'bool': { diff --git a/test/unit/middleware/changeLanguage.js b/test/unit/middleware/changeLanguage.js index 7536810c..ecdb40d0 100644 --- a/test/unit/middleware/changeLanguage.js +++ b/test/unit/middleware/changeLanguage.js @@ -1,4 +1,3 @@ - const setup = require('../../../middleware/changeLanguage'); const proxyquire = require('proxyquire').noCallThru(); const _ = require('lodash'); diff --git a/test/unit/middleware/interpolate.js b/test/unit/middleware/interpolate.js index d90210ef..3602592c 100644 --- a/test/unit/middleware/interpolate.js +++ b/test/unit/middleware/interpolate.js @@ -1,4 +1,3 @@ - const setup = require('../../../middleware/interpolate'); const proxyquire = require('proxyquire').noCallThru(); const _ = require('lodash'); diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index 96fdc1d8..e039437e 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -1,4 +1,3 @@ - const proxyquire = require('proxyquire'); var customConfig = { diff --git a/test/unit/middleware/requestLanguage.js b/test/unit/middleware/requestLanguage.js index 6252e589..e57a8974 100644 --- a/test/unit/middleware/requestLanguage.js +++ b/test/unit/middleware/requestLanguage.js @@ -1,4 +1,3 @@ - var middleware = require('../../../middleware/requestLanguage'); module.exports.tests = {}; diff --git a/test/unit/query/MockQuery.js b/test/unit/query/MockQuery.js index 2cbcf78d..f20dc569 100644 --- a/test/unit/query/MockQuery.js +++ b/test/unit/query/MockQuery.js @@ -1,4 +1,3 @@ - module.exports = class MockQuery { constructor() { this._score_functions = []; diff --git a/test/unit/query/autocomplete_defaults.js b/test/unit/query/autocomplete_defaults.js index 7ccb17a0..69d18c1d 100644 --- a/test/unit/query/autocomplete_defaults.js +++ b/test/unit/query/autocomplete_defaults.js @@ -1,4 +1,3 @@ - var defaults = require('../../../query/autocomplete_defaults'); module.exports.tests = {}; diff --git a/test/unit/query/reverse_defaults.js b/test/unit/query/reverse_defaults.js index 48a9805f..2ea89cbf 100644 --- a/test/unit/query/reverse_defaults.js +++ b/test/unit/query/reverse_defaults.js @@ -1,4 +1,3 @@ - var defaults = require('../../../query/reverse_defaults'); module.exports.tests = {}; diff --git a/test/unit/query/search_defaults.js b/test/unit/query/search_defaults.js index 54b116b7..81dabed3 100644 --- a/test/unit/query/search_defaults.js +++ b/test/unit/query/search_defaults.js @@ -1,4 +1,3 @@ - var defaults = require('../../../query/search_defaults'); module.exports.tests = {}; diff --git a/test/unit/sanitizer/_geo_reverse.js b/test/unit/sanitizer/_geo_reverse.js index 0afc2b15..69fab8fb 100644 --- a/test/unit/sanitizer/_geo_reverse.js +++ b/test/unit/sanitizer/_geo_reverse.js @@ -1,4 +1,3 @@ - const sanitizer = require('../../../sanitizer/_geo_reverse')(); const defaults = require('../../../query/reverse_defaults'); diff --git a/test/unit/sanitizer/wrap.js b/test/unit/sanitizer/wrap.js index 3339b51c..143ee65a 100644 --- a/test/unit/sanitizer/wrap.js +++ b/test/unit/sanitizer/wrap.js @@ -1,4 +1,3 @@ - var wrap = require('../../../sanitizer/wrap'); module.exports.tests = {}; diff --git a/test/unit/schema.js b/test/unit/schema.js index 93e93964..d26a7bf0 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -1,4 +1,3 @@ - const Joi = require('joi'); const schema = require('../../schema'); const _ = require('lodash'); From 922fc240493132eb695cd585896e5ce19fa58971 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 22 Mar 2018 05:20:52 +0000 Subject: [PATCH 44/61] chore(package): update semantic-release to version 15.1.4 Closes #1094 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20e4d849..7765bc36 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "pelias-mock-logger": "1.2.0", "precommit-hook": "^3.0.0", "proxyquire": "^1.7.10", - "semantic-release": "^8.0.0", + "semantic-release": "^15.1.4", "source-map": "^0.7.0", "tap-dot": "1.0.5", "tape": "^4.5.1", From c4330c8af192c0451ac2f5cc3f62ea37cd8e86e3 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Sat, 3 Mar 2018 02:28:08 +0000 Subject: [PATCH 45/61] chore(package): update proxyquire to version 2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7765bc36..50f7d81f 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "nsp": "^3.0.0", "pelias-mock-logger": "1.2.0", "precommit-hook": "^3.0.0", - "proxyquire": "^1.7.10", + "proxyquire": "^2.0.0", "semantic-release": "^15.1.4", "source-map": "^0.7.0", "tap-dot": "1.0.5", From 5ace10adc672d7285a9b0188f4b6d4ad4ddd0f5e Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:45:45 -0400 Subject: [PATCH 46/61] Update README header We should probably propagate this change to all READMEs. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fd42a7f6..73fe3e30 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ >This repository is part of the [Pelias](https://github.com/pelias/pelias) ->project. Pelias is an open-source, open-data geocoder built by ->[Mapzen](https://www.mapzen.com/) that also powers [Mapzen Search](https://mapzen.com/projects/search). Our ->official user documentation is [here](https://mapzen.com/documentation/search/). +>project. Pelias is an open-source, open-data geocoder originally sponsored by +>[Mapzen](https://www.mapzen.com/). Our official user documentation is +>[here](https://github.com/pelias/documentation). # Pelias API Server From 04a3ae84db4f17985e6073933e503cb34231cc22 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:47:06 -0400 Subject: [PATCH 47/61] Update list of services with libpostal --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73fe3e30..863d8dc3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This is the API server for the Pelias project. It's the service that runs to pro ## Documentation -See the [Mapzen Search documentation](https://mapzen.com/documentation/search/). +Full documentation for the Pelias API lives in the [pelias/documentation](https://github.com/pelias/documentation) repository. ## Install Dependencies @@ -43,7 +43,7 @@ The API recognizes the following properties under the top-level `api` key in you |`indexName`|*no*|*pelias*|name of the Elasticsearch index to be used when building queries| |`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable |`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.| -|`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), and [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| +|`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal) and [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| |`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point Example configuration file would look something like this: From 8ac8adb819a91ee5d473ddc4f982ae4230fb6e5e Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:48:04 -0400 Subject: [PATCH 48/61] Fix grammar --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 863d8dc3..03edfc3d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ The API recognizes the following properties under the top-level `api` key in you |`indexName`|*no*|*pelias*|name of the Elasticsearch index to be used when building queries| |`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable |`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.| -|`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal) and [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| +|`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| |`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point Example configuration file would look something like this: From 66e21b4d23eb48b5fd652be5e309870fcae359a0 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:48:24 -0400 Subject: [PATCH 49/61] Update example pelias.json --- README.md | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 03edfc3d..f6232a1b 100644 --- a/README.md +++ b/README.md @@ -46,37 +46,30 @@ The API recognizes the following properties under the top-level `api` key in you |`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| |`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point -Example configuration file would look something like this: +A good starting configuration file includes this section (fill in the service and Elasticsearch hosts as needed): ``` { "esclient": { - "keepAlive": true, - "requestTimeout": "1200000", - "hosts": [ - { - "protocol": "http", - "host": "somesemachine.elb.amazonaws.com", - "port": 9200 - } - ] + "hosts": [{ + "host": "elasticsearch" + }] }, "api": { - "host": "localhost:3100/v1/", - "indexName": "foobar", - "relativeScores": true, "services": { - "pip": { - "url": "http://mypipservice.com:3000" - }, "placeholder": { - "url": "http://myplaceholderservice.com:5000" + "url": "http://placeholder:4100" + }, + "libpostal": { + "url": "http://libpostal:8080" + }, + "pip": { + "url": "http://pip-service:4200" }, "interpolation": { - "url": "http://myinterpolationservice.com:3000", - "timeout": 2500 + "url": "http://interpolation:4300" } - } + }, "defaultParameters": { "focus.point.lat": 12.121212, "focus.point.lon": 21.212121 From 0a1b34aea5f4c817aeb80cc8453f5ee71aab8aa6 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:49:40 -0400 Subject: [PATCH 50/61] Add note that there are services besides Elasticsearch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6232a1b..788f7520 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # Pelias API Server -This is the API server for the Pelias project. It's the service that runs to process user HTTP requests and return results as GeoJSON by querying Elasticsearch. +This is the API server for the Pelias project. It's the service that runs to process user HTTP requests and return results as GeoJSON by querying Elasticsearch and the other Pelias services. [![NPM](https://nodei.co/npm/pelias-api.png?downloads=true&stars=true)](https://nodei.co/npm/pelias-api) From 6718373d873f658af2c320335c6b637be9372c7c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 11:56:12 -0400 Subject: [PATCH 51/61] Update LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index d2a640a9..cb9226ae 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Mapzen +Copyright (c) 2014 Pelias Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. From f8771ca6f85a70a00ef10425eed25f7480261efb Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 5 Apr 2018 12:11:28 -0400 Subject: [PATCH 52/61] Update Node.js versions in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 788f7520..1b03664a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Full documentation for the Pelias API lives in the [pelias/documentation](https: ## Install Dependencies -Note: Pelias requires Node.js v4 or newer +Note: Pelias requires Node.js v6 or newer ```bash npm install @@ -127,7 +127,7 @@ $ curl localhost:9200/pelias/_count?pretty ### Continuous Integration -Travis tests every release against Node.js versions `4` and `6`. +Travis tests every release against all supported Node.js versions. [![Build Status](https://travis-ci.org/pelias/api.png?branch=master)](https://travis-ci.org/pelias/api) From 0785f28a640db2a4729232a6ac54b6cba6db7b45 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 9 Apr 2018 10:10:41 -0400 Subject: [PATCH 53/61] Remove default focus point from example config I predict this would have caused weird query issues for people who copied it --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 1b03664a..0991c143 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,6 @@ A good starting configuration file includes this section (fill in the service an "interpolation": { "url": "http://interpolation:4300" } - }, - "defaultParameters": { - "focus.point.lat": 12.121212, - "focus.point.lon": 21.212121 } }, "logger": { From 1b00b643f1d4f8a11a719c3f8f529d5626b25552 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 24 Apr 2018 14:02:09 -0700 Subject: [PATCH 54/61] Fix link to documentation I think this was broken by https://github.com/pelias/documentation/pull/219 and the rename of pelias/pelias-doc to pelias/documentation --- public/apiDoc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/apiDoc.md b/public/apiDoc.md index 4e3ce186..e0026593 100644 --- a/public/apiDoc.md +++ b/public/apiDoc.md @@ -1 +1 @@ -## [View our documentation on GitHub](https://github.com/pelias/pelias-doc/blob/master/index.md) +## [View our documentation on GitHub](https://github.com/pelias/documentation/) From a0aa5ed7f24cd512ebd395d45955b56875b680dc Mon Sep 17 00:00:00 2001 From: semhul Date: Fri, 27 Apr 2018 14:30:09 +0200 Subject: [PATCH 55/61] Adding support for unit field in structured search. If libpostal finds it in input use it for searching --- controller/structured_libpostal.js | 33 ++++++++----- test/unit/controller/structured_libpostal.js | 49 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/controller/structured_libpostal.js b/controller/structured_libpostal.js index 626bf945..f9362471 100644 --- a/controller/structured_libpostal.js +++ b/controller/structured_libpostal.js @@ -3,19 +3,20 @@ const Debug = require('../helper/debug'); const debugLog = new Debug('controller:libpostal'); const logger = require('pelias-logger').get('api'); -// if there's a house_number in the libpostal response, return it -// otherwise return the postcode field (which may be undefined) -function findHouseNumberField(response) { - const house_number_field = response.find(f => f.label === 'house_number'); - - if (house_number_field) { - return house_number_field; +// Find field in libpostal response +function findField(response, field, replacementField) { + const libpostalField = response.find(f => f.label === field); + + if (libpostalField) { + return libpostalField; + } else if(replacementField) { + return response.find(f => f.label === replacementField); + } else { + return; } - - return response.find(f => f.label === 'postcode'); - } + function setup(libpostalService, should_execute) { function controller( req, res, next ){ // bail early if req/res don't pass conditions for execution @@ -35,7 +36,9 @@ function setup(libpostalService, should_execute) { // libpostal parses some inputs, like `3370 cobbe ave`, as a postcode+street // so because we're treating the entire field as a street address, it's safe // to assume that an identified postcode is actually a house number. - const house_number_field = findHouseNumberField(response); + // if there's a house_number in the libpostal response, return it + // otherwise return the postcode field (which may be undefined) + const house_number_field = findField(response, 'house_number', 'postcode'); // if we're fairly certain that libpostal identified a house number // (from either the house_number or postcode field), place it into the @@ -48,6 +51,14 @@ function setup(libpostalService, should_execute) { // remove the first instance of the number and trim whitespace req.clean.parsed_text.street = _.trim(_.replace(req.clean.parsed_text.address, req.clean.parsed_text.number, '')); + // If libpostal have parsed unit then add it for search + const unit_field = findField(response, 'unit'); + if(unit_field) { + req.clean.parsed_text.unit = unit_field.value; + // Removing unit from street and trim + req.clean.parsed_text.street = _.trim(_.replace(req.clean.parsed_text.street, req.clean.parsed_text.unit, '')); + } + } else { // otherwise no house number was identifiable, so treat the entire input // as a street diff --git a/test/unit/controller/structured_libpostal.js b/test/unit/controller/structured_libpostal.js index 142d8260..16660459 100644 --- a/test/unit/controller/structured_libpostal.js +++ b/test/unit/controller/structured_libpostal.js @@ -291,6 +291,55 @@ module.exports.tests.success_conditions = (test, common) => { }); + test('service returning house_number and unit should set req.clean.parsed_text.unit', t => { + const service = (req, callback) => { + const response = [ + { + label: 'road', + value: 'the street' + }, + { + label: 'house_number', + value: '22' + }, + { + label: 'unit', + value: '1 th' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + parsed_text: { + address: 'the street 22 1 th' + } + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + parsed_text: { + street: 'the street', + number: '22', + unit: '1 th' + } + }, + errors: [] + }, 'req should have been modified'); + + t.end(); + + }); + + }); + // test('service returning valid response should convert and append', t => { // const service = (req, callback) => { // const response = [ From 37430d5daba3a2b5cf6c9a830df6dc6e744e99cc Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 May 2018 14:21:24 -0400 Subject: [PATCH 56/61] Run semantic-relase on the production branch This change will make semantic-release run only on the `production` branch. This means only merges to production will create new NPM packages, GitHub releases, git tags, and `:latest` Docker images on Docker hub. Connects https://github.com/pelias/pelias/issues/721 --- package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 50f7d81f..1b8081e4 100644 --- a/package.json +++ b/package.json @@ -90,5 +90,8 @@ "validate", "test", "check-dependencies" - ] + ], + "release": { + "branch": "production" + } } From b829fc1a2038bd52efd0b7d2a6fbecef4d275777 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 25 Apr 2018 21:09:56 -0400 Subject: [PATCH 57/61] feat: Add Node.js 10 to TravisCI Connects https://github.com/pelias/pelias/issues/723 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f91c3dfc..02cecfe8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ notifications: node_js: - 6 - 8 + - 10 matrix: fast_finish: true script: npm run travis From 06f7099598ccc8e124a69f1c27aaf219da789e58 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 May 2018 15:11:51 -0400 Subject: [PATCH 58/61] Fix semantic release run --- .travis.yml | 2 +- package.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 02cecfe8..52a610d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: before_script: - npm prune after_success: - - npm run semantic-release + - npx semantic-release branches: except: - /^v\d+\.\d+\.\d+$/ diff --git a/package.json b/package.json index 1b8081e4..a7759a27 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "travis": "npm run check-dependencies && npm test", "unit": "./bin/units", "validate": "npm ls", - "semantic-release": "semantic-release pre && npm publish && semantic-release post", "config": "node -e \"console.log(JSON.stringify(require( 'pelias-config' ).generate(require('./schema')), null, 2))\"", "check-dependencies": "node_modules/.bin/npm-check --production --ignore pelias-interpolation", "prune": "npm prune" From 01ca6e6c65d62f66ee3003a68a6df57f0bdd6f31 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 May 2018 19:26:30 -0400 Subject: [PATCH 59/61] Remove npm-check --- package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a7759a27..ffa18966 100644 --- a/package.json +++ b/package.json @@ -14,11 +14,10 @@ "lint": "jshint .", "start": "./bin/start", "test": "npm run unit", - "travis": "npm run check-dependencies && npm test", + "travis": "npm test", "unit": "./bin/units", "validate": "npm ls", "config": "node -e \"console.log(JSON.stringify(require( 'pelias-config' ).generate(require('./schema')), null, 2))\"", - "check-dependencies": "node_modules/.bin/npm-check --production --ignore pelias-interpolation", "prune": "npm prune" }, "repository": { @@ -71,7 +70,6 @@ "difflet": "^1.0.1", "istanbul": "^0.4.2", "jshint": "^2.5.6", - "npm-check": "git://github.com/orangejulius/npm-check.git#disable-update-check", "nsp": "^3.0.0", "pelias-mock-logger": "1.2.0", "precommit-hook": "^3.0.0", @@ -87,8 +85,7 @@ "lint", "prune", "validate", - "test", - "check-dependencies" + "test" ], "release": { "branch": "production" From e5afa25b5ea6b7a3645b643e2368a791a68745f8 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 May 2018 19:28:28 -0400 Subject: [PATCH 60/61] Skip npm prune on Travis Travis always installs fresh, so this should never be needed --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52a610d1..5aec4536 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,6 @@ matrix: script: npm run travis before_install: - npm i -g npm -before_script: - - npm prune after_success: - npx semantic-release branches: From 27385da8ddd551b6e571d660f5fdc5b06c2beb46 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Fri, 11 May 2018 19:40:09 -0400 Subject: [PATCH 61/61] Change default NPM package name --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ffa18966..34af521a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pelias-api", - "version": "0.0.0-semantic-release", + "version": "0.0.0-development", "author": "pelias", "description": "Pelias API", "homepage": "https://github.com/pelias/api",