From 488285f82953bd85152e33a1eb4f2e0cfa723df0 Mon Sep 17 00:00:00 2001 From: Diana Shkolnikov Date: Thu, 9 Nov 2017 18:22:55 -0500 Subject: [PATCH] move route creation to separate files --- controller/index.js | 11 + controller/predicates/index.js | 36 +++ middleware/index.js | 22 ++ query/index.js | 9 + routes/routers/attribution.js | 10 + routes/routers/autocomplete.js | 33 +++ routes/routers/index.js | 10 + routes/routers/nearby.js | 36 +++ routes/routers/place.js | 26 ++ routes/routers/reverse.js | 58 +++++ routes/routers/search.js | 163 ++++++++++++ routes/routers/status.js | 8 + routes/routers/structured.js | 39 +++ routes/routers/utils.js | 48 ++++ routes/v1.js | 463 ++------------------------------- sanitizer/index.js | 9 + service/index.js | 33 +++ 17 files changed, 578 insertions(+), 436 deletions(-) create mode 100644 controller/index.js create mode 100644 controller/predicates/index.js create mode 100644 middleware/index.js create mode 100644 query/index.js create mode 100644 routes/routers/attribution.js create mode 100644 routes/routers/autocomplete.js create mode 100644 routes/routers/index.js create mode 100644 routes/routers/nearby.js create mode 100644 routes/routers/place.js create mode 100644 routes/routers/reverse.js create mode 100644 routes/routers/search.js create mode 100644 routes/routers/status.js create mode 100644 routes/routers/structured.js create mode 100644 routes/routers/utils.js create mode 100644 sanitizer/index.js create mode 100644 service/index.js diff --git a/controller/index.js b/controller/index.js new file mode 100644 index 00000000..8f0832d6 --- /dev/null +++ b/controller/index.js @@ -0,0 +1,11 @@ +module.exports = { + coarse_reverse: require('./coarse_reverse'), + mdToHTML: require('./markdownToHtml'), + libpostal: require('./libpostal'), + place: require('./place'), + placeholder: require('./placeholder'), + search: require('./search'), + search_with_ids: require('./search_with_ids'), + search_with_appending_results: require('./search_with_appending_results'), + status: require('./status') +}; diff --git a/controller/predicates/index.js b/controller/predicates/index.js new file mode 100644 index 00000000..a6e917fa --- /dev/null +++ b/controller/predicates/index.js @@ -0,0 +1,36 @@ +const all = require('predicates').all; +const any = require('predicates').any; +const not = require('predicates').not; + +const hasResponseData = require('./has_response_data'); +const hasRequestErrors = require('./has_request_errors'); +const hasParsedTextProperties = require('./has_parsed_text_properties'); +const hasResultsAtLayers = require('./has_results_at_layers'); + +module.exports = { + hasResponseData: hasResponseData, + hasResultsAtLayers: hasResultsAtLayers, + hasRequestErrors: hasRequestErrors, + hasRequestFocusPoint: require('./has_request_focus_point'), + isCoarseReverse: require('./is_coarse_reverse'), + isAdminOnlyAnalysis: require('./is_admin_only_analysis'), + isAddressItParse: require('./is_addressit_parse'), + hasRequestCategories: require('./has_request_parameter')('categories'), + isOnlyNonAdminLayers: require('./is_only_non_admin_layers'), + + // this can probably be more generalized + isRequestSourcesOnlyWhosOnFirst: require('./is_request_sources_only_whosonfirst'), + hasRequestParameter: require('./has_request_parameter'), + hasParsedTextProperties: hasParsedTextProperties, + isSingleFieldAnalysis: require('./is_single_field_analysis'), + isVenueLayerRequested: require('./is_layer_requested')('venue'), + + // shorthand for standard early-exit conditions + hasResponseDataOrRequestErrors: any(hasResponseData, hasRequestErrors), + hasAdminOnlyResults: not(hasResultsAtLayers.any(['address', 'street'])), + + hasNumberButNotStreet: all( + hasParsedTextProperties.any('number'), + not(hasParsedTextProperties.any('street')) + ) +}; diff --git a/middleware/index.js b/middleware/index.js new file mode 100644 index 00000000..ce4405e4 --- /dev/null +++ b/middleware/index.js @@ -0,0 +1,22 @@ +module.exports = { + trimByGranularity: require('./trimByGranularity'), + trimByGranularityStructured: require('./trimByGranularityStructured'), + distances: require('./distance'), + confidenceScores: require('./confidenceScore'), + confidenceScoresFallback: require('./confidenceScoreFallback'), + confidenceScoresReverse: require('./confidenceScoreReverse'), + accuracy: require('./accuracy'), + dedupe: require('./dedupe'), + interpolate: require('./interpolate'), + localNamingConventions: require('./localNamingConventions'), + renamePlacenames: require('./renamePlacenames'), + geocodeJSON: require('./geocodeJSON'), + sendJSON: require('./sendJSON'), + parseBoundingBox: require('./parseBBox'), + normalizeParentIds: require('./normalizeParentIds'), + assignLabels: require('./assignLabels'), + changeLanguage: require('./changeLanguage'), + sortResponseData: require('./sortResponseData'), + calcSize: require('./sizeCalculator'), + requestLanguage: require('./requestLanguage') +}; \ No newline at end of file diff --git a/query/index.js b/query/index.js new file mode 100644 index 00000000..8240f9e8 --- /dev/null +++ b/query/index.js @@ -0,0 +1,9 @@ +module.exports = { + cascading_fallback: require('./search'), + very_old_prod: require('./search_original'), + structured_geocoding: require('./structured_geocoding'), + reverse: require('./reverse'), + autocomplete: require('./autocomplete'), + address_using_ids: require('./address_search_using_ids'), + venues: require('./venues') +}; diff --git a/routes/routers/attribution.js b/routes/routers/attribution.js new file mode 100644 index 00000000..acbd9055 --- /dev/null +++ b/routes/routers/attribution.js @@ -0,0 +1,10 @@ +const controllers = require('../../controller'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig) => { + return utils.createRouter([ + controllers.mdToHTML(peliasConfig.api, './public/attribution.md') + ]); +}; diff --git a/routes/routers/autocomplete.js b/routes/routers/autocomplete.js new file mode 100644 index 00000000..bb2b66a3 --- /dev/null +++ b/routes/routers/autocomplete.js @@ -0,0 +1,33 @@ +const not = require('predicates').not; + +const sanitizers = require('../../sanitizer'); +const predicates = require('../../controller/predicates'); +const controllers = require('../../controller'); +const queries = require('../../query'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig, esclient, services) => { + + const changeLanguageShouldExecute = utils.changeLanguageShouldExecute(services); + + return [ + sanitizers.autocomplete.middleware(peliasConfig.api), + postProc.requestLanguage, + controllers.search(peliasConfig.api, esclient, queries.autocomplete, not(predicates.hasResponseDataOrRequestErrors)), + postProc.distances('focus.point.'), + postProc.confidenceScores(peliasConfig.api), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, changeLanguageShouldExecute), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]; +}; \ No newline at end of file diff --git a/routes/routers/index.js b/routes/routers/index.js new file mode 100644 index 00000000..2df0419e --- /dev/null +++ b/routes/routers/index.js @@ -0,0 +1,10 @@ +const controllers = require('../../controller'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig) => { + return utils.createRouter([ + controllers.mdToHTML(peliasConfig.api, './public/apiDoc.md') + ]); +}; \ No newline at end of file diff --git a/routes/routers/nearby.js b/routes/routers/nearby.js new file mode 100644 index 00000000..cccb8f3e --- /dev/null +++ b/routes/routers/nearby.js @@ -0,0 +1,36 @@ +const not = require('predicates').not; + +const sanitizers = require('../../sanitizer'); +const predicates = require('../../controller/predicates'); +const controllers = require('../../controller'); +const queries = require('../../query'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig, esclient, services) => { + + const changeLanguageShouldExecute = utils.changeLanguageShouldExecute(services); + + return utils.createRouter([ + sanitizers.nearby.middleware, + postProc.requestLanguage, + postProc.calcSize(), + controllers.search(peliasConfig.api, esclient, queries.reverse, not(predicates.hasResponseDataOrRequestErrors)), + postProc.distances('point.'), + // reverse confidence scoring depends on distance from origin + // so it must be calculated first + postProc.confidenceScoresReverse(), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, changeLanguageShouldExecute), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]); +}; \ No newline at end of file diff --git a/routes/routers/place.js b/routes/routers/place.js new file mode 100644 index 00000000..5407f56e --- /dev/null +++ b/routes/routers/place.js @@ -0,0 +1,26 @@ +const sanitizers = require('../../sanitizer'); +const controllers = require('../../controller'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig, esclient, services) => { + + const changeLanguageShouldExecute = utils.changeLanguageShouldExecute(services); + + return utils.createRouter([ + sanitizers.place.middleware, + postProc.requestLanguage, + controllers.place(peliasConfig.api, esclient), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, changeLanguageShouldExecute), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]); +}; \ No newline at end of file diff --git a/routes/routers/reverse.js b/routes/routers/reverse.js new file mode 100644 index 00000000..9abe1ba8 --- /dev/null +++ b/routes/routers/reverse.js @@ -0,0 +1,58 @@ +const not = require('predicates').not; +const any = require('predicates').any; +const all = require('predicates').all; + +const sanitizers = require('../../sanitizer'); +const predicates = require('../../controller/predicates'); +const controllers = require('../../controller'); +const queries = require('../../query'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig, esclient, services) => { + + const changeLanguageShouldExecute = utils.changeLanguageShouldExecute(services); + + // execute under the following conditions: + // - there are no errors or data + // - request is not coarse OR pip service is disabled + const nonCoarseReverseShouldExecute = all( + not(predicates.hasResponseDataOrRequestErrors), + any( + not(predicates.isCoarseReverse), + not(services.pip.isEnabled) + ) + ); + + // fallback to coarse reverse when regular reverse didn't return anything + const coarseReverseShouldExecute = all( + services.pip.isEnabled, + not(predicates.hasRequestErrors), + not(predicates.hasResponseData) + ); + + + return [ + sanitizers.reverse.middleware, + postProc.requestLanguage, + postProc.calcSize(), + controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute), + controllers.coarse_reverse(services.pip.service, coarseReverseShouldExecute), + postProc.distances('point.'), + // reverse confidence scoring depends on distance from origin + // so it must be calculated first + postProc.confidenceScoresReverse(), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, changeLanguageShouldExecute), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]; +}; \ No newline at end of file diff --git a/routes/routers/search.js b/routes/routers/search.js new file mode 100644 index 00000000..6be0c2fd --- /dev/null +++ b/routes/routers/search.js @@ -0,0 +1,163 @@ +const all = require('predicates').all; +const any = require('predicates').any; +const not = require('predicates').not; + +const sanitizers = require('../../sanitizer'); +const predicates = require('../../controller/predicates'); +const controllers = require('../../controller'); +const queries = require('../../query'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + +module.exports.create = (peliasConfig, esclient, services) => { + + // helpers to replace vague booleans + const geometricFiltersApply = true; + const geometricFiltersDontApply = false; + + return utils.createRouter([ + sanitizers.search.middleware(peliasConfig.api), + postProc.requestLanguage, + postProc.calcSize(), + controllers.libpostal(libpostalShouldExecute()), + controllers.placeholder(services.placeholder.service, geometricFiltersApply, placeholderGeodisambiguationShouldExecute(services)), + controllers.placeholder(services.placeholder.service, geometricFiltersDontApply, placeholderIdsLookupShouldExecute(services)), + controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute()), + controllers.search_with_appending_results(peliasConfig.api, esclient, queries.venues, venuesSearchShouldExecute()), + // 3rd parameter is which query module to use, use fallback first, then + // use original search strategy if first query didn't return anything + controllers.search(peliasConfig.api, esclient, queries.cascading_fallback, fallbackQueryShouldExecute(services)), + sanitizers.defer_to_addressit(shouldDeferToAddressIt(services)), + controllers.search(peliasConfig.api, esclient, queries.very_old_prod, oldProdQueryShouldExecute()), + postProc.trimByGranularity(), + postProc.distances('focus.point.'), + postProc.confidenceScores(peliasConfig.api), + postProc.confidenceScoresFallback(), + postProc.interpolate(services.interpolation.service, utils.interpolationShouldExecute(services)), + postProc.sortResponseData(require('pelias-sorting'), predicates.hasAdminOnlyResults), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, utils.changeLanguageShouldExecute(services)), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]); +}; + +// search for venues under the following conditions: +// - there are no request errors +// - analysis is only admin (no address, query, or street) +// - there's a single field in analysis +// - request has a focus.point available +// - TODO: needs check for venues is in layers +// https://github.com/pelias/pelias/issues/564 +const venuesSearchShouldExecute = () => { + return all( + not(predicates.hasRequestErrors), + predicates.isVenueLayerRequested, + predicates.isAdminOnlyAnalysis, + predicates.isSingleFieldAnalysis, + predicates.hasRequestFocusPoint + ); +}; + +const libpostalShouldExecute = () => { + return all( + not(predicates.hasRequestErrors), + not(predicates.isRequestSourcesOnlyWhosOnFirst) + ); +}; + +const searchWithIdsShouldExecute = () => { + return all( + not(predicates.hasRequestErrors), + // don't search-with-ids if there's a query or category + not(predicates.hasParsedTextProperties.any('query', 'category')), + // there must be a street + predicates.hasParsedTextProperties.any('street') + ); +}; + + +// execute placeholder if libpostal only parsed as admin-only and needs to +// be geodisambiguated +const placeholderGeodisambiguationShouldExecute = (services) => { + return all( + not(predicates.hasResponseDataOrRequestErrors), + services.placeholder.isEnabled, + // check request.clean for several conditions first + not( + any( + // layers only contains venue, address, or street + predicates.isOnlyNonAdminLayers, + // don't geodisambiguate if categories were requested + predicates.hasRequestCategories + ) + ), + any( + // only geodisambiguate if libpostal returned only admin areas or libpostal was skipped + predicates.isAdminOnlyAnalysis, + predicates.isRequestSourcesOnlyWhosOnFirst + ) + ); +}; + +// execute placeholder if libpostal identified address parts but ids need to +// be looked up for admin parts +const placeholderIdsLookupShouldExecute = (services) => { + return all( + not(predicates.hasResponseDataOrRequestErrors), + services.placeholder.isEnabled, + // check clean.parsed_text for several conditions that must all be true + all( + // run placeholder if clean.parsed_text has 'street' + predicates.hasParsedTextProperties.any('street'), + // don't run placeholder if there's a query or category + not(predicates.hasParsedTextProperties.any('query', 'category')), + // run placeholder if there are any admin areas identified + predicates.hasParsedTextProperties.any('neighbourhood', 'borough', 'city', 'county', 'state', 'country') + ) + ); +}; + +// placeholder should have executed, useful for determining whether to actually +// fallback or not (don't fallback to old search if the placeholder response +// should be honored as is) +const placeholderShouldHaveExecuted = (services) => { + return any( + placeholderGeodisambiguationShouldExecute(services), + placeholderIdsLookupShouldExecute(services) + ); +}; + +// don't execute the cascading fallback query IF placeholder should have executed +// that way, if placeholder didn't return anything, don't try to find more things the old way +const fallbackQueryShouldExecute = (services) => { + return all( + not(predicates.hasRequestErrors), + not(predicates.hasResponseData), + not(placeholderShouldHaveExecuted(services)) + ); +}; + +// defer to addressit for analysis IF there's no response AND placeholder should not have executed +const shouldDeferToAddressIt = (services) => { + return all( + not(predicates.hasRequestErrors), + not(predicates.hasResponseData), + not(placeholderShouldHaveExecuted(services)) + ); +}; + +// call very old prod query if addressit was the parser +const oldProdQueryShouldExecute = () => { + return all( + not(predicates.hasRequestErrors), + predicates.isAddressItParse + ); +}; \ No newline at end of file diff --git a/routes/routers/status.js b/routes/routers/status.js new file mode 100644 index 00000000..e29bd5ba --- /dev/null +++ b/routes/routers/status.js @@ -0,0 +1,8 @@ +const controllers = require('../../controller'); + +const utils = require('./utils'); + + +module.exports.create = () => { + return utils.createRouter([ controllers.status ]); +}; \ No newline at end of file diff --git a/routes/routers/structured.js b/routes/routers/structured.js new file mode 100644 index 00000000..85d92a83 --- /dev/null +++ b/routes/routers/structured.js @@ -0,0 +1,39 @@ +const all = require('predicates').all; +const not = require('predicates').not; + +const sanitizers = require('../../sanitizer'); +const predicates = require('../../controller/predicates'); +const controllers = require('../../controller'); +const queries = require('../../query'); +const postProc = require('../../middleware'); + +const utils = require('./utils'); + + +module.exports.create = (peliasConfig, esclient, services) => { + + const interpolationShouldExecute = utils.interpolationShouldExecute(services); + const changeLanguageShouldExecute = utils.changeLanguageShouldExecute(services); + + return [ + sanitizers.structured_geocoding.middleware(peliasConfig.api), + postProc.requestLanguage, + postProc.calcSize(), + controllers.search(peliasConfig.api, esclient, queries.structured_geocoding, not(predicates.hasResponseDataOrRequestErrors)), + postProc.trimByGranularityStructured(), + postProc.distances('focus.point.'), + postProc.confidenceScores(peliasConfig.api), + postProc.confidenceScoresFallback(), + postProc.interpolate(services.interpolation.service, interpolationShouldExecute), + postProc.dedupe(), + postProc.accuracy(), + postProc.localNamingConventions(), + postProc.renamePlacenames(), + postProc.parseBoundingBox(), + postProc.normalizeParentIds(), + postProc.changeLanguage(services.language.service, changeLanguageShouldExecute), + postProc.assignLabels(), + postProc.geocodeJSON(peliasConfig.api, utils.base), + postProc.sendJSON + ]; +}; \ No newline at end of file diff --git a/routes/routers/utils.js b/routes/routers/utils.js new file mode 100644 index 00000000..1b708348 --- /dev/null +++ b/routes/routers/utils.js @@ -0,0 +1,48 @@ +const Router = require('express').Router; + +const all = require('predicates').all; +const not = require('predicates').not; + +const predicates = require('../../controller/predicates'); + +module.exports = { + base: '/v1/', + + /** + * Helper function for creating routers + * + * @param {[{function}]} functions + * @returns {express.Router} + */ + createRouter: (functions) => { + const router = Router(); // jshint ignore:line + functions.forEach(function (f) { + router.use(f); + }); + return router; + }, + + // interpolate if: + // - there's a number and street + // - there are street-layer results (these are results that need to be interpolated) + interpolationShouldExecute: (services) => { + return all( + not(predicates.hasRequestErrors), + services.interpolation.isEnabled, + predicates.hasParsedTextProperties.all('number', 'street'), + predicates.hasResultsAtLayers.any('street') + ); + }, + + // get language adjustments if: + // - there's a response + // - theres's a lang parameter in req.clean + changeLanguageShouldExecute: (services) => { + return all( + predicates.hasResponseData, + not(predicates.hasRequestErrors), + services.language.isEnabled, + predicates.hasRequestParameter('lang') + ); + } +}; \ No newline at end of file diff --git a/routes/v1.js b/routes/v1.js index bb802f4c..e5503e5a 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -1,108 +1,7 @@ -var Router = require('express').Router; -var elasticsearch = require('elasticsearch'); +const elasticsearch = require('elasticsearch'); -const all = require('predicates').all; -const any = require('predicates').any; -const not = require('predicates').not; -const _ = require('lodash'); - -/** ----------------------- sanitizers ----------------------- **/ -var sanitizers = { - autocomplete: require('../sanitizer/autocomplete'), - place: require('../sanitizer/place'), - search: require('../sanitizer/search'), - defer_to_addressit: require('../sanitizer/defer_to_addressit'), - structured_geocoding: require('../sanitizer/structured_geocoding'), - reverse: require('../sanitizer/reverse'), - nearby: require('../sanitizer/nearby') -}; - -/** ----------------------- middleware ------------------------ **/ -var middleware = { - calcSize: require('../middleware/sizeCalculator'), - requestLanguage: require('../middleware/requestLanguage') -}; - -/** ----------------------- controllers ----------------------- **/ - -var controllers = { - coarse_reverse: require('../controller/coarse_reverse'), - mdToHTML: require('../controller/markdownToHtml'), - libpostal: require('../controller/libpostal'), - structured_libpostal: require('../controller/structured_libpostal'), - place: require('../controller/place'), - placeholder: require('../controller/placeholder'), - search: require('../controller/search'), - search_with_ids: require('../controller/search_with_ids'), - search_with_appending_results: require('../controller/search_with_appending_results'), - status: require('../controller/status') -}; - -var queries = { - cascading_fallback: require('../query/search'), - very_old_prod: require('../query/search_original'), - structured_geocoding: require('../query/structured_geocoding'), - reverse: require('../query/reverse'), - autocomplete: require('../query/autocomplete'), - address_using_ids: require('../query/address_search_using_ids'), - venues: require('../query/venues') -}; - -/** ----------------------- controllers ----------------------- **/ - -var postProc = { - trimByGranularity: require('../middleware/trimByGranularity'), - trimByGranularityStructured: require('../middleware/trimByGranularityStructured'), - distances: require('../middleware/distance'), - confidenceScores: require('../middleware/confidenceScore'), - confidenceScoresFallback: require('../middleware/confidenceScoreFallback'), - confidenceScoresReverse: require('../middleware/confidenceScoreReverse'), - accuracy: require('../middleware/accuracy'), - dedupe: require('../middleware/dedupe'), - interpolate: require('../middleware/interpolate'), - localNamingConventions: require('../middleware/localNamingConventions'), - renamePlacenames: require('../middleware/renamePlacenames'), - geocodeJSON: require('../middleware/geocodeJSON'), - sendJSON: require('../middleware/sendJSON'), - parseBoundingBox: require('../middleware/parseBBox'), - normalizeParentIds: require('../middleware/normalizeParentIds'), - assignLabels: require('../middleware/assignLabels'), - changeLanguage: require('../middleware/changeLanguage'), - sortResponseData: require('../middleware/sortResponseData') -}; - -// predicates that drive whether controller/search runs -const hasResponseData = require('../controller/predicates/has_response_data'); -const hasRequestFocusPoint = require('../controller/predicates/has_request_focus_point'); -const hasRequestErrors = require('../controller/predicates/has_request_errors'); -const isCoarseReverse = require('../controller/predicates/is_coarse_reverse'); -const isAdminOnlyAnalysis = require('../controller/predicates/is_admin_only_analysis'); -const hasResultsAtLayers = require('../controller/predicates/has_results_at_layers'); -const isAddressItParse = require('../controller/predicates/is_addressit_parse'); -const hasRequestCategories = require('../controller/predicates/has_request_parameter')('categories'); -const isOnlyNonAdminLayers = require('../controller/predicates/is_only_non_admin_layers'); -// this can probably be more generalized -const isRequestSourcesOnlyWhosOnFirst = require('../controller/predicates/is_request_sources_only_whosonfirst'); -const hasRequestParameter = require('../controller/predicates/has_request_parameter'); -const hasParsedTextProperties = require('../controller/predicates/has_parsed_text_properties'); -const isSingleFieldAnalysis = require('../controller/predicates/is_single_field_analysis'); -const isVenueLayerRequested = require('../controller/predicates/is_layer_requested')('venue'); - -// shorthand for standard early-exit conditions -const hasResponseDataOrRequestErrors = any(hasResponseData, hasRequestErrors); -const hasAdminOnlyResults = not(hasResultsAtLayers.any(['address', 'street'])); - -const hasNumberButNotStreet = all( - hasParsedTextProperties.any('number'), - not(hasParsedTextProperties.any('street')) -); - -const serviceWrapper = require('pelias-microservice-wrapper').service; -const PlaceHolder = require('../service/configurations/PlaceHolder'); -const PointInPolygon = require('../service/configurations/PointInPolygon'); -const Language = require('../service/configurations/Language'); -const Interpolation = require('../service/configurations/Interpolation'); -const Libpostal = require('../service/configurations/Libpostal'); +const serviceWrappers = require('../service'); +const utils = require('./routers/utils'); /** * Append routes to app @@ -111,348 +10,40 @@ const Libpostal = require('../service/configurations/Libpostal'); * @param {object} peliasConfig */ function addRoutes(app, peliasConfig) { - const esclient = elasticsearch.Client(peliasConfig.esclient); - - const pipConfiguration = new PointInPolygon(_.defaultTo(peliasConfig.api.services.pip, {})); - const pipService = serviceWrapper(pipConfiguration); - const isPipServiceEnabled = _.constant(pipConfiguration.isEnabled()); - - const placeholderConfiguration = new PlaceHolder(_.defaultTo(peliasConfig.api.services.placeholder, {})); - const placeholderService = serviceWrapper(placeholderConfiguration); - const isPlaceholderServiceEnabled = _.constant(placeholderConfiguration.isEnabled()); - - const changeLanguageConfiguration = new Language(_.defaultTo(peliasConfig.api.services.placeholder, {})); - const changeLanguageService = serviceWrapper(changeLanguageConfiguration); - const isChangeLanguageEnabled = _.constant(changeLanguageConfiguration.isEnabled()); - - const interpolationConfiguration = new Interpolation(_.defaultTo(peliasConfig.api.services.interpolation, {})); - const interpolationService = serviceWrapper(interpolationConfiguration); - const isInterpolationEnabled = _.constant(interpolationConfiguration.isEnabled()); - - // standard libpostal should use req.clean.text for the `address` parameter - const libpostalConfiguration = new Libpostal( - _.defaultTo(peliasConfig.api.services.libpostal, {}), - _.property('clean.text')); - const libpostalService = serviceWrapper(libpostalConfiguration); - - // structured libpostal should use req.clean.parsed_text.address for the `address` parameter - const structuredLibpostalConfiguration = new Libpostal( - _.defaultTo(peliasConfig.api.services.libpostal, {}), - _.property('clean.parsed_text.address')); - const structuredLibpostalService = serviceWrapper(structuredLibpostalConfiguration); - - // fallback to coarse reverse when regular reverse didn't return anything - const coarseReverseShouldExecute = all( - isPipServiceEnabled, not(hasRequestErrors), not(hasResponseData) - ); - - const libpostalShouldExecute = all( - not(hasRequestErrors), - not(isRequestSourcesOnlyWhosOnFirst) - ); - // for libpostal to execute for structured requests, req.clean.parsed_text.address must exist - const structuredLibpostalShouldExecute = all( - not(hasRequestErrors), - hasParsedTextProperties.all('address') - ); - - // execute placeholder if libpostal only parsed as admin-only and needs to - // be geodisambiguated - const placeholderGeodisambiguationShouldExecute = all( - not(hasResponseDataOrRequestErrors), - isPlaceholderServiceEnabled, - // check request.clean for several conditions first - not( - any( - // layers only contains venue, address, or street - isOnlyNonAdminLayers, - // don't geodisambiguate if categories were requested - hasRequestCategories - ) - ), - any( - // only geodisambiguate if libpostal returned only admin areas or libpostal was skipped - isAdminOnlyAnalysis, - isRequestSourcesOnlyWhosOnFirst - ) - ); - - // execute placeholder if libpostal identified address parts but ids need to - // be looked up for admin parts - const placeholderIdsLookupShouldExecute = all( - not(hasResponseDataOrRequestErrors), - isPlaceholderServiceEnabled, - // check clean.parsed_text for several conditions that must all be true - all( - // run placeholder if clean.parsed_text has 'street' - hasParsedTextProperties.any('street'), - // don't run placeholder if there's a query or category - not(hasParsedTextProperties.any('query', 'category')), - // run placeholder if there are any adminareas identified - hasParsedTextProperties.any('neighbourhood', 'borough', 'city', 'county', 'state', 'country') - ) - ); - - const searchWithIdsShouldExecute = all( - not(hasRequestErrors), - // don't search-with-ids if there's a query or category - not(hasParsedTextProperties.any('query', 'category')), - // there must be a street - hasParsedTextProperties.any('street') - ); - - // placeholder should have executed, useful for determining whether to actually - // fallback or not (don't fallback to old search if the placeholder response - // should be honored as is) - const placeholderShouldHaveExecuted = any( - placeholderGeodisambiguationShouldExecute, - placeholderIdsLookupShouldExecute - ); - - // don't execute the cascading fallback query IF placeholder should have executed - // that way, if placeholder didn't return anything, don't try to find more things the old way - const fallbackQueryShouldExecute = all( - not(hasRequestErrors), - not(hasResponseData), - not(placeholderShouldHaveExecuted) - ); - - // defer to addressit for analysis IF there's no response AND placeholder should not have executed - const shouldDeferToAddressIt = all( - not(hasRequestErrors), - not(hasResponseData), - not(placeholderShouldHaveExecuted) - ); - - // call very old prod query if addressit was the parser - const oldProdQueryShouldExecute = all( - not(hasRequestErrors), - isAddressItParse - ); - - // get language adjustments if: - // - there's a response - // - theres's a lang parameter in req.clean - const changeLanguageShouldExecute = all( - hasResponseData, - not(hasRequestErrors), - isChangeLanguageEnabled, - hasRequestParameter('lang') - ); - - // interpolate if: - // - there's a number and street - // - there are street-layer results (these are results that need to be interpolated) - const interpolationShouldExecute = all( - not(hasRequestErrors), - isInterpolationEnabled, - hasParsedTextProperties.all('number', 'street'), - hasResultsAtLayers.any('street') - ); - - // execute under the following conditions: - // - there are no errors or data - // - request is not coarse OR pip service is disabled - const nonCoarseReverseShouldExecute = all( - not(hasResponseDataOrRequestErrors), - any( - not(isCoarseReverse), - not(isPipServiceEnabled) - ) - ); - - // search for venues under the following conditions: - // - there are no request errors - // - analysis is only admin (no address, query, or street) - // - there's a single field in analysis - // - request has a focus.point available - // - TODO: needs check for venues is in layers - // https://github.com/pelias/pelias/issues/564 - const venuesSearchShouldExecute = all( - not(hasRequestErrors), - isVenueLayerRequested, - isAdminOnlyAnalysis, - isSingleFieldAnalysis, - hasRequestFocusPoint - ); - - // helpers to replace vague booleans - const geometricFiltersApply = true; - const geometricFiltersDontApply = false; - - var base = '/v1/'; - - /** ------------------------- routers ------------------------- **/ + const esclient = elasticsearch.Client(peliasConfig.esclient); - var routers = { - index: createRouter([ - controllers.mdToHTML(peliasConfig.api, './public/apiDoc.md') - ]), - attribution: createRouter([ - controllers.mdToHTML(peliasConfig.api, './public/attribution.md') - ]), - search: createRouter([ - sanitizers.search.middleware(peliasConfig.api), - middleware.requestLanguage, - middleware.calcSize(), - controllers.libpostal(libpostalService, libpostalShouldExecute), - controllers.placeholder(placeholderService, geometricFiltersApply, placeholderGeodisambiguationShouldExecute), - controllers.placeholder(placeholderService, geometricFiltersDontApply, placeholderIdsLookupShouldExecute), - controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute), - controllers.search_with_appending_results(peliasConfig.api, esclient, queries.venues, venuesSearchShouldExecute), - // 3rd parameter is which query module to use, use fallback first, then - // use original search strategy if first query didn't return anything - controllers.search(peliasConfig.api, esclient, queries.cascading_fallback, fallbackQueryShouldExecute), - sanitizers.defer_to_addressit(shouldDeferToAddressIt), - controllers.search(peliasConfig.api, esclient, queries.very_old_prod, oldProdQueryShouldExecute), - postProc.trimByGranularity(), - postProc.distances('focus.point.'), - postProc.confidenceScores(peliasConfig.api), - postProc.confidenceScoresFallback(), - postProc.interpolate(interpolationService, interpolationShouldExecute), - postProc.sortResponseData(require('pelias-sorting'), hasAdminOnlyResults), - postProc.dedupe(), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - structured: createRouter([ - sanitizers.structured_geocoding.middleware(peliasConfig.api), - middleware.requestLanguage, - middleware.calcSize(), - controllers.structured_libpostal(structuredLibpostalService, structuredLibpostalShouldExecute), - controllers.search(peliasConfig.api, esclient, queries.structured_geocoding, not(hasResponseDataOrRequestErrors)), - postProc.trimByGranularityStructured(), - postProc.distances('focus.point.'), - postProc.confidenceScores(peliasConfig.api), - postProc.confidenceScoresFallback(), - postProc.interpolate(interpolationService, interpolationShouldExecute), - postProc.dedupe(), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - autocomplete: createRouter([ - sanitizers.autocomplete.middleware(peliasConfig.api), - middleware.requestLanguage, - controllers.search(peliasConfig.api, esclient, queries.autocomplete, not(hasResponseDataOrRequestErrors)), - postProc.distances('focus.point.'), - postProc.confidenceScores(peliasConfig.api), - postProc.dedupe(), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - reverse: createRouter([ - sanitizers.reverse.middleware, - middleware.requestLanguage, - middleware.calcSize(), - controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute), - controllers.coarse_reverse(pipService, coarseReverseShouldExecute), - postProc.distances('point.'), - // reverse confidence scoring depends on distance from origin - // so it must be calculated first - postProc.confidenceScoresReverse(), - postProc.dedupe(), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - nearby: createRouter([ - sanitizers.nearby.middleware, - middleware.requestLanguage, - middleware.calcSize(), - controllers.search(peliasConfig.api, esclient, queries.reverse, not(hasResponseDataOrRequestErrors)), - postProc.distances('point.'), - // reverse confidence scoring depends on distance from origin - // so it must be calculated first - postProc.confidenceScoresReverse(), - postProc.dedupe(), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - place: createRouter([ - sanitizers.place.middleware, - middleware.requestLanguage, - controllers.place(peliasConfig.api, esclient), - postProc.accuracy(), - postProc.localNamingConventions(), - postProc.renamePlacenames(), - postProc.parseBoundingBox(), - postProc.normalizeParentIds(), - postProc.changeLanguage(changeLanguageService, changeLanguageShouldExecute), - postProc.assignLabels(), - postProc.geocodeJSON(peliasConfig.api, base), - postProc.sendJSON - ]), - status: createRouter([ - controllers.status - ]) + const services = serviceWrappers.create(peliasConfig); + + const routers = { + index: require('./routers/index').create(peliasConfig), + attribution: require('./routers/attribution').create(peliasConfig), + search: require('./routers/search').create(peliasConfig, esclient, services), + structured: require('./routers/structured').create(peliasConfig, esclient, services), + autocomplete: require('./routers/autocomplete').create(peliasConfig, esclient, services), + reverse: require('./routers/reverse').create(peliasConfig, esclient, services), + nearby: require('./routers/nearby').create(peliasConfig, esclient, services), + place: require('./routers/place').create(peliasConfig, esclient, services), + status: require('./routers/status').create() }; // static data endpoints - app.get ( base, routers.index ); - app.get ( base + 'attribution', routers.attribution ); - app.get ( '/attribution', routers.attribution ); - app.get ( '/status', routers.status ); + app.get ( utils.base, routers.index ); + app.get ( utils.base + 'attribution', routers.attribution ); + app.get ( '/attribution', routers.attribution ); + app.get ( '/status', routers.status ); // backend dependent endpoints - app.get ( base + 'place', routers.place ); - app.get ( base + 'autocomplete', routers.autocomplete ); - app.get ( base + 'search', routers.search ); - app.post( base + 'search', routers.search ); - app.get ( base + 'search/structured', routers.structured ); - app.get ( base + 'reverse', routers.reverse ); - app.get ( base + 'nearby', routers.nearby ); + app.get ( utils.base + 'place', routers.place ); + app.get ( utils.base + 'autocomplete', routers.autocomplete ); + app.get ( utils.base + 'search', routers.search ); + app.post( utils.base + 'search', routers.search ); + app.get ( utils.base + 'search/structured', routers.structured ); + app.get ( utils.base + 'reverse', routers.reverse ); + app.get ( utils.base + 'nearby', routers.nearby ); } -/** - * Helper function for creating routers - * - * @param {[{function}]} functions - * @returns {express.Router} - */ -function createRouter(functions) { - var router = Router(); // jshint ignore:line - functions.forEach(function (f) { - router.use(f); - }); - return router; -} - module.exports.addRoutes = addRoutes; diff --git a/sanitizer/index.js b/sanitizer/index.js new file mode 100644 index 00000000..afa89f19 --- /dev/null +++ b/sanitizer/index.js @@ -0,0 +1,9 @@ +module.exports = { + autocomplete: require('./autocomplete'), + place: require('./place'), + search: require('./search'), + defer_to_addressit: require('./defer_to_addressit'), + structured_geocoding: require('./structured_geocoding'), + reverse: require('./reverse'), + nearby: require('./nearby') +}; diff --git a/service/index.js b/service/index.js new file mode 100644 index 00000000..83bbd63c --- /dev/null +++ b/service/index.js @@ -0,0 +1,33 @@ +const _ = require('lodash'); + +const serviceWrapper = require('pelias-microservice-wrapper').service; +const PlaceHolder = require('./configurations/PlaceHolder'); +const PointInPolygon = require('./configurations/PointInPolygon'); +const Language = require('./configurations/Language'); +const Interpolation = require('./configurations/Interpolation'); + +module.exports.create = (peliasConfig) => { + const pipConfiguration = new PointInPolygon(_.defaultTo(peliasConfig.api.services.pip, {})); + const placeholderConfiguration = new PlaceHolder(_.defaultTo(peliasConfig.api.services.placeholder, {})); + const changeLanguageConfiguration = new Language(_.defaultTo(peliasConfig.api.services.placeholder, {})); + const interpolationConfiguration = new Interpolation(_.defaultTo(peliasConfig.api.services.interpolation, {})); + + return { + pip: { + service: serviceWrapper(pipConfiguration), + isEnabled: _.constant(pipConfiguration.isEnabled()) + }, + placeholder: { + service: serviceWrapper(placeholderConfiguration), + isEnabled: _.constant(placeholderConfiguration.isEnabled()) + }, + language: { + service: serviceWrapper(changeLanguageConfiguration), + isEnabled: _.constant(changeLanguageConfiguration.isEnabled()) + }, + interpolation: { + service: serviceWrapper(interpolationConfiguration), + isEnabled: _.constant(interpolationConfiguration.isEnabled()) + } + }; +};