diff --git a/.circleci/docker.sh b/.circleci/docker.sh index 1acd51cb..0dfa433c 100644 --- a/.circleci/docker.sh +++ b/.circleci/docker.sh @@ -6,22 +6,19 @@ DATE=`date +%Y-%m-%d` DOCKER_REPOSITORY="pelias" DOCKER_PROJECT="${DOCKER_REPOSITORY}/${CIRCLE_PROJECT_REPONAME}" -# skip builds on greenkeeper branches -if [[ -z "${CIRCLE_BRANCH##*greenkeeper*}" ]]; then - exit 0 -fi +BRANCH="$(echo $CIRCLE_BRANCH | tr '/' '-')" #slashes are not valid in docker tags. replace with dashes # the name of the image that represents the "branch", that is an image that will be updated over time with the git branch # the production branch is changed to "latest", otherwise the git branch becomes the name of the version -if [[ "${CIRCLE_BRANCH}" == "production" ]]; then +if [[ "${BRANCH}" == "production" ]]; then DOCKER_BRANCH_IMAGE_VERSION="latest" else - DOCKER_BRANCH_IMAGE_VERSION="${CIRCLE_BRANCH}" + DOCKER_BRANCH_IMAGE_VERSION="$BRANCH" fi DOCKER_BRANCH_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_BRANCH_IMAGE_VERSION}" # the name of the image that represents the "tag", that is an image that is named with the date and git commit and will never be changed -DOCKER_TAG_IMAGE_VERSION="${CIRCLE_BRANCH}-${DATE}-${CIRCLE_SHA1}" +DOCKER_TAG_IMAGE_VERSION="${BRANCH}-${DATE}-${CIRCLE_SHA1}" DOCKER_TAG_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_TAG_IMAGE_VERSION}" # build image and login to docker hub diff --git a/README.md b/README.md index 0991c143..d90f834c 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ The API recognizes the following properties under the top-level `api` key in you |`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.| |`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| |`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point +|`targets.layers_by_source`
`targets.source_aliases`
`targets.layer_aliases`|no | |custom values for which `sources` and `layers` the API accepts ([more info](https://github.com/pelias/api/pull/1131)). +|`attributionURL`|no| (autodetedted)|The full URL to use for the attribution link returned in all Pelias responses. Pelias will attempt to autodetect this host, but it will often be correct if, for example, there is a proxy between Pelias and its users. This parameter allows setting a specific URL to avoid any such issues| A good starting configuration file includes this section (fill in the service and Elasticsearch hosts as needed): diff --git a/controller/predicates/is_request_sources_includes_whosonfirst.js b/controller/predicates/is_request_sources_includes_whosonfirst.js new file mode 100644 index 00000000..fe3fcd8b --- /dev/null +++ b/controller/predicates/is_request_sources_includes_whosonfirst.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); +const Debug = require('../../helper/debug'); +const debugLog = new Debug('controller:predicates:is_request_sources_includes_whosonfirst'); +const stackTraceLine = require('../../helper/stackTraceLine'); + +// returns true IFF 'whosonfirst' is included in the requested sources +module.exports = (req, res) => { + const is_request_sources_includes_whosonfirst = _.get(req, 'clean.sources', []).includes( + 'whosonfirst' + ); + debugLog.push(req, () => ({ + reply: is_request_sources_includes_whosonfirst, + stack_trace: stackTraceLine() + })); + return is_request_sources_includes_whosonfirst; +}; diff --git a/controller/predicates/is_request_sources_undefined.js b/controller/predicates/is_request_sources_undefined.js new file mode 100644 index 00000000..77e8afea --- /dev/null +++ b/controller/predicates/is_request_sources_undefined.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); +const Debug = require('../../helper/debug'); +const debugLog = new Debug('controller:predicates:is_request_sources_undefined'); +const stackTraceLine = require('../../helper/stackTraceLine'); + +// returns true IFF there are no requested sources +module.exports = (req, res) => { + const is_request_sources_undefined = _.isEmpty( + _.get(req, 'clean.sources') + ); + debugLog.push(req, () => ({ + reply: is_request_sources_undefined, + stack_trace: stackTraceLine() + })); + return is_request_sources_undefined; +}; diff --git a/helper/TypeMapping.js b/helper/TypeMapping.js new file mode 100644 index 00000000..97d87a26 --- /dev/null +++ b/helper/TypeMapping.js @@ -0,0 +1,146 @@ +const _ = require('lodash'); +const elasticsearch = require('elasticsearch'); + +var TypeMapping = function(){ + + // A list of all sources + this.sources = []; + + // A list of alternate names for sources, mostly used to save typing + this.source_aliases = {}; + + // A list of all layers + this.layers = []; + + /* + * A list of all layers in each source. This is used for convenience elswhere + * and to determine when a combination of source and layer parameters is + * not going to match any records and will return no results. + */ + this.layers_by_source = {}; + + /* + * A list of layer aliases that can be used to support specific use cases + * (like coarse geocoding) * or work around the fact that different sources + * may have layers that mean the same thing but have a different name + */ + this.layer_aliases = {}; + + /* + * An object that contains all sources or aliases. The key is the source or alias, + * the value is either that source, or the canonical name for that alias if it's an alias. + */ + this.source_mapping = {}; + + /* + * An object that has a key for each possible layer or alias, + * and returns either that layer, or gall the layers in the alias + */ + this.layer_mapping = {}; +}; + +TypeMapping.addStandardTargetsToAliases = function(standard, aliases) { + var combined = _.extend({}, aliases); + standard.forEach(function(target) { + if (combined[target] === undefined) { + combined[target] = [target]; + } + }); + + return combined; +}; + +// source alias setter +TypeMapping.prototype.setSourceAliases = function( aliases ){ + this.source_aliases = aliases; +}; + +// layers-by-source alias setter +TypeMapping.prototype.setLayersBySource = function( lbs ){ + this.layers_by_source = lbs; +}; + +// layer alias setter +TypeMapping.prototype.setLayerAliases = function( aliases ){ + this.layer_aliases = aliases; +}; + +// generate mappings after setters have been run +TypeMapping.prototype.generateMappings = function(){ + this.sources = Object.keys( this.layers_by_source ); + this.source_mapping = TypeMapping.addStandardTargetsToAliases(this.sources, this.source_aliases); + this.layers = _.uniq(Object.keys(this.layers_by_source).reduce(function(acc, key) { + return acc.concat(this.layers_by_source[key]); + }.bind(this), [])); + this.layer_mapping = TypeMapping.addStandardTargetsToAliases(this.layers, this.layer_aliases); +}; + +// load values from targets block +TypeMapping.prototype.loadTargets = function( targetsBlock ){ + + if( !_.isObject(targetsBlock) ){ return; } + + // set values from targets block + this.setSourceAliases( targetsBlock.source_aliases || {} ); + this.setLayersBySource( targetsBlock.layers_by_source || {} ); + this.setLayerAliases( targetsBlock.layer_aliases || {} ); + + // generate the mappings + this.generateMappings(); +}; + +// load values from either pelias config file or from elasticsearch +TypeMapping.prototype.load = function( done ){ + + // load pelias config + const peliasConfigTargets = _.get( + require('pelias-config').generate(require('../schema')), + 'api.targets', {} + ); + + // load targets from config file + this.loadTargets( peliasConfigTargets ); + + // do not load values from elasticsearch + if( true !== peliasConfigTargets.auto_discover ){ + if( 'function' === typeof done ){ done(); } + return; + } + + if( 'function' === typeof done ){ done(); } + return; + + // load values from elasticsearch + + // create connection to elasticsearch + // const esclient = elasticsearch.Client(peliasConfig.esclient); + + // const query = { + // requestCache: true, + // preference: '_replica_first', + // timeout: '10s', + // body: { + // aggs: { + // sources: { + // terms: { + // field: 'source', + // size: 100 + // } + // }, + // layers: { + // terms: { + // field: 'layer', + // size: 100 + // } + // } + // }, + // size: 0 + // } + // }; + + // esclient.search( query, ( err, res ) => { + // console.error( err, res ); + // }); +}; + +module.exports = TypeMapping; diff --git a/helper/type_mapping.js b/helper/type_mapping.js index 41303583..987aa26a 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -1,86 +1,8 @@ -const _ = require('lodash'); +const TypeMapping = require('./TypeMapping'); -function addStandardTargetsToAliases(standard, aliases) { - var combined = _.extend({}, aliases); - standard.forEach(function(target) { - if (combined[target] === undefined) { - combined[target] = [target]; - } - }); +// instantiate a new type mapping +var tm = new TypeMapping(); +tm.load(); - return combined; -} - -/* - * Sources - */ - -// a list of all sources -var SOURCES = ['openstreetmap', 'openaddresses', 'geonames', 'whosonfirst']; - -/* - * A list of alternate names for sources, mostly used to save typing - */ -var SOURCE_ALIASES = { - 'osm': ['openstreetmap'], - 'oa': ['openaddresses'], - 'gn': ['geonames'], - 'wof': ['whosonfirst'] -}; - -/* - * Create an object that contains all sources or aliases. The key is the source or alias, - * the value is either that source, or the canonical name for that alias if it's an alias. - */ -var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES); - -/* - * Layers - */ - -/* - * A list of all layers in each source. This is used for convenience elswhere - * and to determine when a combination of source and layer parameters is - * not going to match any records and will return no results. - */ -var LAYERS_BY_SOURCE = { - openstreetmap: [ 'address', 'venue', 'street' ], - openaddresses: [ 'address' ], - geonames: [ 'country','macroregion', 'region', 'county','localadmin', - 'locality','borough', 'neighbourhood', 'venue' ], - whosonfirst: [ 'continent', 'empire', 'country', 'dependency', 'macroregion', 'region', - 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', - 'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode', - 'continent', 'ocean', 'marinearea'] -}; - -/* - * A list of layer aliases that can be used to support specific use cases - * (like coarse geocoding) * or work around the fact that different sources - * may have layers that mean the same thing but have a different name - */ -var LAYER_ALIASES = { - 'coarse': [ 'continent', 'empire', 'country', 'dependency', 'macroregion', - 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', - 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode', - 'continent', 'ocean', 'marinearea'] -}; - -// create a list of all layers by combining each entry from LAYERS_BY_SOURCE -var LAYERS = _.uniq(Object.keys(LAYERS_BY_SOURCE).reduce(function(acc, key) { - return acc.concat(LAYERS_BY_SOURCE[key]); -}, [])); - -/* - * Create the an object that has a key for each possible layer or alias, - * and returns either that layer, or all the layers in the alias - */ -var LAYER_MAPPING = addStandardTargetsToAliases(LAYERS, LAYER_ALIASES); - -module.exports = { - sources: SOURCES, - layers: LAYERS, - source_mapping: SOURCE_MAPPING, - layer_mapping: LAYER_MAPPING, - layers_by_source: LAYERS_BY_SOURCE -}; +// export singleton +module.exports = tm; \ No newline at end of file diff --git a/middleware/geocodeJSON.js b/middleware/geocodeJSON.js index 0ff2e303..60c74f68 100644 --- a/middleware/geocodeJSON.js +++ b/middleware/geocodeJSON.js @@ -47,10 +47,10 @@ function convertToGeocodeJSON(req, res, next, opts) { // OPTIONAL. Default: null. The attribution of the data. In case of multiple sources, // and then multiple attributions, can be an object with one key by source. // Can be a URI on the server, which outlines attribution details. - res.body.geocoding.attribution = url.format({ + res.body.geocoding.attribution = opts.config.attributionURL || url.format({ protocol: req.protocol, host: req.get('host'), - pathname: opts.basePath + 'attribution' + pathname: 'attribution' }); // OPTIONAL. Default: null. The query that has been issued to trigger the diff --git a/middleware/interpolate.js b/middleware/interpolate.js index 10c3f7e8..8507709a 100644 --- a/middleware/interpolate.js +++ b/middleware/interpolate.js @@ -2,6 +2,7 @@ const async = require('async'); const logger = require( 'pelias-logger' ).get( 'api' ); const source_mapping = require('../helper/type_mapping').source_mapping; const _ = require('lodash'); +const stable = require('stable'); /** example response from interpolation web service: @@ -116,7 +117,7 @@ function setup(service, should_execute) { // sort the results to ensure that addresses show up higher than street centroids if (_.has(res, 'data')) { - res.data.sort((a, b) => { + res.data = stable(res.data, (a, b) => { if (a.layer === 'address' && b.layer !== 'address') { return -1; } if (a.layer !== 'address' && b.layer === 'address') { return 1; } return 0; diff --git a/package.json b/package.json index 34af521a..9c6e977e 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "homepage": "https://github.com/pelias/api", "license": "MIT", "main": "index.js", + "bin": "./bin/start", "scripts": { "audit": "npm shrinkwrap; node node_modules/nsp/bin/nsp check; rm npm-shrinkwrap.json;", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", @@ -36,32 +37,33 @@ "node": ">=6.0.0" }, "dependencies": { + "@mapbox/geojson-extent": "^0.3.1", "addressit": "1.5.0", "async": "^2.0.0", "check-types": "^7.0.0", - "elasticsearch": "^14.2.1", + "elasticsearch": "^15.0.0", "elasticsearch-exceptions": "0.0.4", "express": "^4.8.8", "geojson": "^0.5.0", - "@mapbox/geojson-extent": "^0.3.1", "geolib": "^2.0.18", "iso-639-3": "^1.0.0", "iso3166-1": "^0.3.0", - "joi": "^12.0.0", + "joi": "^13.1.3", "locale": "^0.1.0", "lodash": "^4.17.4", - "markdown": "0.5.0", + "markdown": "^0.5.0", "morgan": "^1.8.2", - "pelias-categories": "1.2.0", - "pelias-config": "2.14.0", - "pelias-labels": "1.8.0", - "pelias-logger": "0.3.1", - "pelias-microservice-wrapper": "1.3.0", - "pelias-model": "5.3.2", - "pelias-query": "9.1.1", - "pelias-sorting": "1.1.0", + "pelias-categories": "^1.2.0", + "pelias-config": "^3.0.2", + "pelias-labels": "^1.8.0", + "pelias-logger": "^0.4.2", + "pelias-microservice-wrapper": "^1.4.0", + "pelias-model": "^5.5.2", + "pelias-query": "^9.1.1", + "pelias-sorting": "^1.2.0", "predicates": "^2.0.0", - "retry": "^0.10.1", + "retry": "^0.12.0", + "stable": "^0.1.8", "stats-lite": "^2.0.4", "through2": "^2.0.3" }, @@ -71,14 +73,14 @@ "istanbul": "^0.4.2", "jshint": "^2.5.6", "nsp": "^3.0.0", - "pelias-mock-logger": "1.2.0", + "pelias-mock-logger": "^1.3.0", "precommit-hook": "^3.0.0", "proxyquire": "^2.0.0", "semantic-release": "^15.1.4", "source-map": "^0.7.0", - "tap-dot": "1.0.5", + "tap-dot": "^2.0.0", "tape": "^4.5.1", - "tmp": "0.0.33", + "tmp": "^0.0.33", "uglify-js": "^3.0.4" }, "pre-commit": [ @@ -88,6 +90,7 @@ "test" ], "release": { - "branch": "production" + "branch": "production", + "success": [] } } diff --git a/public/attribution.md b/public/attribution.md index f2b34bd3..fc96eaa8 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -1,7 +1,7 @@ ## Attribution * Geocoding by [Pelias](https://pelias.io). * Data from - * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) - * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation - * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/) - * [WhosOnFirst](https://www.whosonfirst.org/) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md) + * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/). Also see the [OSM Geocoding Guidelines](https://wiki.osmfoundation.org/wiki/Licence/Community_Guidelines/Geocoding_-_Guideline) for acceptable use. + * [OpenAddresses](http://openaddresses.io) under [various public-domain and share-alike licenses](http://results.openaddresses.io/) + * [GeoNames](http://www.geonames.org/) under [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) + * [WhosOnFirst](https://www.whosonfirst.org/) under [various CC-BY or CC-0 equivalent licenses](https://whosonfirst.org/docs/licenses/) diff --git a/query/autocomplete.js b/query/autocomplete.js index 8d4cf257..f5b37982 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -7,7 +7,6 @@ const logger = require('pelias-logger').get('api'); // additional views (these may be merged in to pelias/query at a later date) var views = { ngrams_strict: require('./view/ngrams_strict'), - focus_selected_layers: require('./view/focus_selected_layers'), ngrams_last_token_only: require('./view/ngrams_last_token_only'), phrase_first_tokens_only: require('./view/phrase_first_tokens_only'), pop_subquery: require('./view/pop_subquery'), @@ -42,7 +41,7 @@ query.score( peliasQuery.view.admin('neighbourhood') ); // scoring boost query.score( views.boost_exact_matches ); -query.score( views.focus_selected_layers( views.ngrams_strict ) ); +query.score( peliasQuery.view.focus( views.ngrams_strict ) ); query.score( peliasQuery.view.popularity( views.pop_subquery ) ); query.score( peliasQuery.view.population( views.pop_subquery ) ); diff --git a/query/view/focus_selected_layers.js b/query/view/focus_selected_layers.js deleted file mode 100644 index 4fda1780..00000000 --- a/query/view/focus_selected_layers.js +++ /dev/null @@ -1,38 +0,0 @@ -var peliasQuery = require('pelias-query'); - -/** - This view is the same as `peliasQuery.view.focus` with one exception: - - if the view is generated successfully, we add a 'filter' clause which - restricts the targeted '_type' to be in the list specified below. - - documents which are not in the '_type' list below will simply score 0 for - this section of the query. -**/ - -module.exports = function( subview ){ - return function( vs ){ - - // don't perform this query on single character inputs - // as its too unperformant to sort a large part of the index. - if( vs.var('input:name').get().length < 2 ){ - return null; - } - - if( !subview ){ return null; } // subview validation failed - var macroView = peliasQuery.view.focus( subview ); - if( !macroView ){ return null; } // macroView validation failed - var view = macroView( vs ); - - if( view && view.hasOwnProperty('function_score') ){ - view.function_score.filter = { - 'or': [ - { 'term': { 'layer': 'venue' } }, - { 'term': { 'layer': 'address' } } - ] - }; - } - - return view; - }; -}; diff --git a/routes/v1.js b/routes/v1.js index 6b40f1d6..79243794 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -80,6 +80,9 @@ const hasRequestCategories = require('../controller/predicates/has_request_param const isOnlyNonAdminLayers = require('../controller/predicates/is_only_non_admin_layers'); // this can probably be more generalized const isRequestSourcesOnlyWhosOnFirst = require('../controller/predicates/is_request_sources_only_whosonfirst'); +const isRequestSourcesIncludesWhosOnFirst = require('../controller/predicates/is_request_sources_includes_whosonfirst'); +const isRequestSourcesUndefined = require('../controller/predicates/is_request_sources_undefined'); + const hasRequestParameter = require('../controller/predicates/has_request_parameter'); const hasParsedTextProperties = require('../controller/predicates/has_parsed_text_properties'); @@ -167,9 +170,14 @@ function addRoutes(app, peliasConfig) { ) ), any( - // only geodisambiguate if libpostal returned only admin areas or libpostal was skipped - isAdminOnlyAnalysis, - isRequestSourcesOnlyWhosOnFirst + isRequestSourcesOnlyWhosOnFirst, + all( + isAdminOnlyAnalysis, + any( + isRequestSourcesUndefined, + isRequestSourcesIncludesWhosOnFirst + ) + ) ) ); @@ -259,7 +267,6 @@ function addRoutes(app, peliasConfig) { // helpers to replace vague booleans const geometricFiltersApply = true; - const geometricFiltersDontApply = false; var base = '/v1/'; @@ -278,7 +285,7 @@ function addRoutes(app, peliasConfig) { middleware.calcSize(), controllers.libpostal(libpostalService, libpostalShouldExecute), controllers.placeholder(placeholderService, geometricFiltersApply, placeholderGeodisambiguationShouldExecute), - controllers.placeholder(placeholderService, geometricFiltersDontApply, placeholderIdsLookupShouldExecute), + controllers.placeholder(placeholderService, geometricFiltersApply, placeholderIdsLookupShouldExecute), controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute), // 3rd parameter is which query module to use, use fallback first, then // use original search strategy if first query didn't return anything diff --git a/sanitizer/_deprecate_quattroshapes.js b/sanitizer/_deprecate_quattroshapes.js deleted file mode 100644 index 1db2fef1..00000000 --- a/sanitizer/_deprecate_quattroshapes.js +++ /dev/null @@ -1,43 +0,0 @@ -var _ = require('lodash'); - -/** - In the process of phasing out the 'quattroshapes' source in favour of 'whosonfirst' - we will emit a warning to users so they can begin upgrading their clients. - - In the interim we will automatically rewrite all requests for quattroshapes to whosonfirst. - - @todo: this is only temporary - @see: https://github.com/pelias/api/issues/442 -**/ - -function _sanitize( raw, clean, opts ) { - // error & warning messages - var messages = { errors: [], warnings: [] }; - - // only applicably when 'sources' param is privided - if( raw.hasOwnProperty('sources') ){ - - var sources = raw.sources.split(','); - if (_.includes(sources, 'quattroshapes') || _.includes(sources, 'qs')) { - - // emit a warning message so users can transition. - messages.warnings.push('You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'); - - // user requested 'quattroshapes', we will give them 'whosonfirst' instead. - sources = _.without(sources, 'quattroshapes', 'qs'); - sources.push('whosonfirst'); - raw.sources = sources.join(','); - } - } - - return messages; -} - -module.exports = () => ({ - sanitize: _sanitize -}); diff --git a/sanitizer/_text.js b/sanitizer/_text.js index 1df63411..bae2d4a1 100644 --- a/sanitizer/_text.js +++ b/sanitizer/_text.js @@ -1,6 +1,9 @@ const check = require('check-types'); const _ = require('lodash'); +// ref: https://en.wikipedia.org/wiki/Quotation_mark +const QUOTES = `"'«»‘’‚‛“”„‟‹›⹂「」『』〝〞〟﹁﹂﹃﹄"'「」`; + // validate texts, convert types and apply defaults function _sanitize( raw, clean ){ @@ -10,12 +13,12 @@ function _sanitize( raw, clean ){ // invalid input 'text' // must call `!check.nonEmptyString` since `check.emptyString` returns // `false` for `undefined` and `null` - if( !check.nonEmptyString( raw.text ) ){ - messages.errors.push('invalid param \'text\': text length, must be >0'); + const text = _.trim( _.trim( raw.text ), QUOTES ); + if( !check.nonEmptyString( text ) ){ + messages.errors.push('invalid param \'text\': text length, must be >0'); } else { - clean.text = raw.text; - + clean.text = text; } return messages; diff --git a/sanitizer/nearby.js b/sanitizer/nearby.js index 1cd3b276..087492de 100644 --- a/sanitizer/nearby.js +++ b/sanitizer/nearby.js @@ -5,7 +5,6 @@ var type_mapping = require('../helper/type_mapping'); var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitizers, must be run after them diff --git a/sanitizer/reverse.js b/sanitizer/reverse.js index ce14e862..c3bb89dc 100644 --- a/sanitizer/reverse.js +++ b/sanitizer/reverse.js @@ -3,7 +3,6 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'), sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitizers, must be run after them diff --git a/sanitizer/search.js b/sanitizer/search.js index 008ae693..4b74bd2d 100644 --- a/sanitizer/search.js +++ b/sanitizer/search.js @@ -5,7 +5,6 @@ module.exports.middleware = (_api_pelias_config) => { var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), text: require('../sanitizer/_text')(), size: require('../sanitizer/_size')(/* use defaults*/), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), diff --git a/sanitizer/structured_geocoding.js b/sanitizer/structured_geocoding.js index 670a26aa..047bb8ff 100644 --- a/sanitizer/structured_geocoding.js +++ b/sanitizer/structured_geocoding.js @@ -6,7 +6,6 @@ module.exports.middleware = (_api_pelias_config) => { var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), synthesize_analysis: require('../sanitizer/_synthesize_analysis')(), iso2_to_iso3: require('../sanitizer/_iso2_to_iso3')(), city_name_standardizer: require('../sanitizer/_city_name_standardizer')(), diff --git a/test/ciao/reverse/sources_deprecation_warning.coffee b/test/ciao/reverse/sources_deprecation_warning.coffee deleted file mode 100644 index 51445800..00000000 --- a/test/ciao/reverse/sources_deprecation_warning.coffee +++ /dev/null @@ -1,34 +0,0 @@ - -#> quattroshapes is being phased out and so should emit a warning message -path: '/v1/reverse?point.lat=1&point.lon=2&sources=qs' - -#? 200 ok -response.statusCode.should.be.equal 200 -response.should.have.header 'charset', 'utf8' -response.should.have.header 'content-type', 'application/json; charset=utf-8' - -#? valid geocoding block -should.exist json.geocoding -should.exist json.geocoding.version -should.exist json.geocoding.attribution -should.exist json.geocoding.query -should.exist json.geocoding.engine -should.exist json.geocoding.engine.name -should.exist json.geocoding.engine.author -should.exist json.geocoding.engine.version -should.exist json.geocoding.timestamp - -#? valid geojson -json.type.should.be.equal 'FeatureCollection' -json.features.should.be.instanceof Array - -#? expected errors -should.not.exist json.geocoding.errors - -#? expected warnings -should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] - -#? inputs -json.geocoding.query['size'].should.eql 10 -json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/ciao/search/sources_deprecation_warning.coffee b/test/ciao/search/sources_deprecation_warning.coffee deleted file mode 100644 index fe82d298..00000000 --- a/test/ciao/search/sources_deprecation_warning.coffee +++ /dev/null @@ -1,35 +0,0 @@ - -#> quattroshapes is being phased out and so should emit a warning message -path: '/v1/search?sources=qs&text=a' - -#? 200 ok -response.statusCode.should.be.equal 200 -response.should.have.header 'charset', 'utf8' -response.should.have.header 'content-type', 'application/json; charset=utf-8' - -#? valid geocoding block -should.exist json.geocoding -should.exist json.geocoding.version -should.exist json.geocoding.attribution -should.exist json.geocoding.query -should.exist json.geocoding.engine -should.exist json.geocoding.engine.name -should.exist json.geocoding.engine.author -should.exist json.geocoding.engine.version -should.exist json.geocoding.timestamp - -#? valid geojson -json.type.should.be.equal 'FeatureCollection' -json.features.should.be.instanceof Array - -#? expected errors -should.not.exist json.geocoding.errors - -#? expected warnings -should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] - -#? inputs -json.geocoding.query['size'].should.eql 10 -json.geocoding.query['text'].should.eql 'a' -json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js b/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js new file mode 100644 index 00000000..97c0d95f --- /dev/null +++ b/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js @@ -0,0 +1,90 @@ +const _ = require('lodash'); +const is_request_sources_includes_whosonfirst = require('../../../../controller/predicates/is_request_sources_includes_whosonfirst'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + t.ok(_.isFunction(is_request_sources_includes_whosonfirst), 'is_request_sources_includes_whosonfirst is a function'); + t.end(); + }); +}; + +module.exports.tests.true_conditions = (test, common) => { + test('sources includes \'whosonfirst\' should return true', (t) => { + const req = { + clean: { + sources: [ + 'whosonfirst', + 'not whosonfirst' + ] + } + }; + + t.ok(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('empty req.clean.sources should return false', (t) => { + const req = { + clean: { + sources: [] + } + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + +}; + +module.exports.tests.false_conditions = (test, common) => { + test('undefined req should return false', (t) => { + t.notOk(is_request_sources_includes_whosonfirst(undefined)); + t.end(); + + }); + + test('undefined req.clean should return false', (t) => { + const req = {}; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('undefined req.clean.sources should return false', (t) => { + const req = { + clean: {} + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('sources not \'whosonfirst\' should return false', (t) => { + const req = { + clean: { + sources: [ + 'not whosonfirst' + ] + } + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + }); +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`GET /is_request_sources_includes_whosonfirst ${name}`, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/controller/predicates/is_request_sources_undefined.js b/test/unit/controller/predicates/is_request_sources_undefined.js new file mode 100644 index 00000000..d7486bb8 --- /dev/null +++ b/test/unit/controller/predicates/is_request_sources_undefined.js @@ -0,0 +1,78 @@ +const _ = require('lodash'); +const is_request_sources_undefined = require('../../../../controller/predicates/is_request_sources_undefined'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + t.ok(_.isFunction(is_request_sources_undefined), 'is_request_sources_undefined is a function'); + t.end(); + }); +}; + +module.exports.tests.true_conditions = (test, common) => { + test('undefined req should return true', (t) => { + + t.ok(is_request_sources_undefined(undefined)); + t.end(); + + }); + + test('undefined req.clean should return true', (t) => { + const req = {}; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + + test('undefined req.clean.sources should return true', (t) => { + const req = { + clean: {} + }; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + + test('empty req.clean.sources should return true', (t) => { + const req = { + clean: { + sources: [] + } + }; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + +}; + +module.exports.tests.false_conditions = (test, common) => { + test('sources not empty should return false', (t) => { + const req = { + clean: { + sources: [ + 'not empty' + ] + } + }; + + t.notOk(is_request_sources_undefined(req)); + t.end(); + + }); + +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`GET /is_request_sources_undefined ${name}`, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 219bea36..2acc0e1d 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -46,21 +46,7 @@ module.exports = { 'weight': 15 }], 'score_mode': 'avg', - 'boost_mode': 'replace', - 'filter': { - 'or': [ - { - 'term': { - 'layer': 'venue' - } - }, - { - 'term': { - 'layer': 'address' - } - } - ] - } + 'boost_mode': 'replace' } },{ 'function_score': { diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 80e44f81..d3029455 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -46,21 +46,7 @@ module.exports = { 'weight': 15 }], 'score_mode': 'avg', - 'boost_mode': 'replace', - 'filter': { - 'or': [ - { - 'term': { - 'layer': 'venue' - } - }, - { - 'term': { - 'layer': 'address' - } - } - ] - } + 'boost_mode': 'replace' } },{ 'function_score': { diff --git a/test/unit/helper/TypeMapping.js b/test/unit/helper/TypeMapping.js new file mode 100644 index 00000000..b4e45475 --- /dev/null +++ b/test/unit/helper/TypeMapping.js @@ -0,0 +1,208 @@ +const _ = require('lodash'); +const TypeMapping = require('../../../helper/TypeMapping'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test) { + test('valid interface', function(t) { + t.equal(typeof TypeMapping, 'function', 'TypeMapping is a function'); + + t.equal(typeof TypeMapping.addStandardTargetsToAliases, 'function', 'addStandardTargetsToAliases() is a function'); + t.equal(typeof TypeMapping.prototype.setSourceAliases, 'function', 'setSourceAliases() is a function'); + + t.equal(typeof TypeMapping.prototype.setLayersBySource, 'function', 'setLayersBySource() is a function'); + t.equal(typeof TypeMapping.prototype.setLayerAliases, 'function', 'setLayerAliases() is a function'); + + t.equal(typeof TypeMapping.prototype.generateMappings, 'function', 'generateMappings() is a function'); + + t.equal(typeof TypeMapping.prototype.loadTargets, 'function', 'loadTargets() is a function'); + t.equal(typeof TypeMapping.prototype.load, 'function', 'load() is a function'); + + t.end(); + }); +}; + +module.exports.tests.constructor = function(test) { + test('constructor', function(t) { + + var doc = new TypeMapping(); + + // initial values + t.deepEqual(doc.sources, [], 'initial value'); + t.deepEqual(doc.source_aliases, {}, 'initial value'); + t.deepEqual(doc.layers, [], 'initial value'); + t.deepEqual(doc.layers_by_source, {}, 'initial value'); + t.deepEqual(doc.layer_aliases, {}, 'initial value'); + t.deepEqual(doc.source_mapping, {}, 'initial value'); + t.deepEqual(doc.layer_mapping, {}, 'initial value'); + + t.end(); + }); +}; + +module.exports.tests.addStandardTargetsToAliases = function(test) { + test('static method addStandardTargetsToAliases', function(t) { + + var aliases = { test: ['test2'] }; + + t.deepEqual( + TypeMapping.addStandardTargetsToAliases([], aliases), + { test: ['test2'] } + ); + t.deepEqual(aliases, aliases, 'aliases object not mutated'); + + t.deepEqual( + TypeMapping.addStandardTargetsToAliases(['test'], aliases), + { test: ['test2'] }, + 'not modified' + ); + t.deepEqual(aliases, aliases, 'aliases object not mutated'); + + t.deepEqual( + TypeMapping.addStandardTargetsToAliases(['baz'], aliases), + { test: ['test2'], baz: ['baz'] } + ); + t.deepEqual(aliases, aliases, 'aliases object not mutated'); + + t.deepEqual( + TypeMapping.addStandardTargetsToAliases(['baz','boo'], aliases), + { test: ['test2'], baz: ['baz'], boo: ['boo'] } + ); + t.deepEqual(aliases, aliases, 'aliases object not mutated'); + + t.end(); + + }); +}; + +module.exports.tests.setSourceAliases = function(test) { + test('setter setSourceAliases', function(t) { + var tm = new TypeMapping(); + t.deepEqual(tm.source_aliases, {}); + tm.setSourceAliases({ foo: ['foo', 'bar'] }); + t.deepEqual(tm.source_aliases, { foo: ['foo', 'bar'] }); + t.end(); + }); +}; + +module.exports.tests.setLayersBySource = function(test) { + test('setter setLayersBySource', function(t) { + var tm = new TypeMapping(); + t.deepEqual(tm.layers_by_source, {}); + tm.setLayersBySource({ foo: ['foo', 'bar'] }); + t.deepEqual(tm.layers_by_source, { foo: ['foo', 'bar'] }); + t.end(); + }); +}; + +module.exports.tests.setLayerAliases = function(test) { + test('setter setLayerAliases', function(t) { + var tm = new TypeMapping(); + t.deepEqual(tm.layer_aliases, {}); + tm.setLayerAliases({ foo: ['foo', 'bar'] }); + t.deepEqual(tm.layer_aliases, { foo: ['foo', 'bar'] }); + t.end(); + }); +}; + +module.exports.tests.generateMappings = function(test) { + test('generateMappings - no-op', function(t) { + var tm = new TypeMapping(); + t.deepEqual(tm.sources, []); + t.deepEqual(tm.source_mapping, {}); + t.deepEqual(tm.layers, []); + t.deepEqual(tm.layer_mapping, {}); + tm.generateMappings(); + t.deepEqual(tm.sources, []); + t.deepEqual(tm.source_mapping, {}); + t.deepEqual(tm.layers, []); + t.deepEqual(tm.layer_mapping, {}); + t.end(); + }); + test('generateMappings - sources', function(t) { + var tm = new TypeMapping(); + tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; + tm.generateMappings(); + t.deepEqual(tm.sources, ['foo', 'faz']); + t.end(); + }); + test('generateMappings - source_mapping', function(t) { + var tm = new TypeMapping(); + tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; + tm.source_aliases = { foo: ['foo','f'], bar: ['bar', 'b'], baz: ['baz'] }; + tm.generateMappings(); + t.deepEqual(tm.source_mapping, { foo: ['foo', 'f'], bar: ['bar', 'b'], baz: ['baz'], faz: ['faz'] }); + t.end(); + }); + test('generateMappings - layers', function(t) { + var tm = new TypeMapping(); + tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; + tm.generateMappings(); + t.deepEqual(tm.layers, ['foo','faz']); + t.end(); + }); + test('generateMappings - layer_mapping', function(t) { + var tm = new TypeMapping(); + tm.layers_by_source = { foo: ['foo'], faz: ['faz'] }; + tm.layer_aliases = { foo: ['foo','f'], bar: ['bar', 'b'], baz: ['baz'] }; + tm.generateMappings(); + t.deepEqual(tm.layer_mapping, { foo: ['foo', 'f'], bar: ['bar', 'b'], baz: ['baz'], faz: ['faz'] }); + t.end(); + }); +}; + +module.exports.tests.loadTargets = function(test) { + test('loadTargets - undefined', function(t) { + var tm = new TypeMapping(); + tm.loadTargets(); + t.deepEqual(tm.sources, []); + t.deepEqual(tm.source_mapping, {}); + t.deepEqual(tm.layers, []); + t.deepEqual(tm.layer_mapping, {}); + t.end(); + }); + test('loadTargets', function(t) { + var tm = new TypeMapping(); + tm.loadTargets({ + source_aliases: { source1: ['s1', 's2'], source2: ['s3', 's4'] }, + layers_by_source: { source1: ['layer1', 'layer3'], source2: ['layer2'] }, + layer_aliases: { layer1: ['l1', 'l2'], layer2: ['l3', 'l4'] }, + }); + t.deepEqual(tm.sources, [ 'source1', 'source2' ]); + t.deepEqual(tm.source_mapping, { source1: [ 's1', 's2' ], source2: [ 's3', 's4' ] }); + t.deepEqual(tm.layers, [ 'layer1', 'layer3', 'layer2' ]); + t.deepEqual(tm.layer_mapping, { layer1: [ 'l1', 'l2' ], layer2: [ 'l3', 'l4' ], layer3: [ 'layer3' ] }); + t.end(); + }); +}; + +module.exports.tests.load = function(test) { + test('load from pelias config', function(t) { + var tm = new TypeMapping(); + tm.load(() => { + + // load pelias config + const expected = _.get( + require('pelias-config').generate(require('../../../schema')), + 'api.targets', {} + ); + + // values copied from config + t.deepEqual(tm.layers_by_source, expected.layers_by_source || {}); + t.deepEqual(tm.source_aliases, expected.source_aliases || {}); + t.deepEqual(tm.layer_aliases, expected.layer_aliases || {}); + t.end(); + }); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('TypeMapping: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/helper/type_mapping.js b/test/unit/helper/type_mapping.js index ec8cd379..25b06d0f 100644 --- a/test/unit/helper/type_mapping.js +++ b/test/unit/helper/type_mapping.js @@ -4,18 +4,31 @@ var type_mapping = require('../../../helper/type_mapping'); module.exports.tests = {}; module.exports.tests.interfaces = function(test, common) { - test('basic layer mapping', function(t) { - t.deepEquals(type_mapping.layer_mapping.venue, ['venue']); - t.deepEquals(type_mapping.layer_mapping.address, ['address']); + + test('complete sources', function(t) { + t.deepEquals(type_mapping.sources, [ 'openstreetmap', 'openaddresses', 'geonames', 'whosonfirst' ]); t.end(); }); - test('alias layer mapping', function(t) { - t.deepEquals(type_mapping.layer_mapping.coarse, - [ 'continent', 'empire', 'country', 'dependency', 'macroregion', - 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', - 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode', - 'continent', 'ocean', 'marinearea']); + test('complete layers', function(t) { + t.deepEquals(type_mapping.layers, [ 'address', 'venue', 'street', 'country', 'macroregion', + 'region', 'county', 'localadmin', 'locality', 'borough', 'neighbourhood', 'continent', + 'empire', 'dependency', 'macrocounty', 'macrohood', 'microhood', 'disputed', + 'postalcode', 'ocean', 'marinearea' ]); + t.end(); + }); + + test('complete source mapping', function(t) { + t.deepEquals(type_mapping.source_mapping, { + osm: [ 'openstreetmap' ], + oa: [ 'openaddresses' ], + gn: [ 'geonames' ], + wof: [ 'whosonfirst' ], + openstreetmap: [ 'openstreetmap' ], + openaddresses: [ 'openaddresses' ], + geonames: [ 'geonames' ], + whosonfirst: [ 'whosonfirst' ] + }); t.end(); }); @@ -30,6 +43,66 @@ module.exports.tests.interfaces = function(test, common) { t.deepEquals(type_mapping.source_mapping.wof, ['whosonfirst']); t.end(); }); + + test('complete layer mapping', function(t) { + t.deepEquals(type_mapping.layer_mapping, { + coarse: [ 'continent', 'empire', 'country', 'dependency', 'macroregion', 'region', + 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', + 'neighbourhood', 'microhood', 'disputed', 'postalcode', 'continent', 'ocean', 'marinearea' + ], + address: [ 'address' ], + venue: [ 'venue' ], + street: [ 'street' ], + country: [ 'country' ], + macroregion: [ 'macroregion' ], + region: [ 'region' ], + county: [ 'county' ], + localadmin: [ 'localadmin' ], + locality: [ 'locality' ], + borough: [ 'borough' ], + neighbourhood: [ 'neighbourhood' ], + continent: [ 'continent' ], + empire: [ 'empire' ], + dependency: [ 'dependency' ], + macrocounty: [ 'macrocounty' ], + macrohood: [ 'macrohood' ], + microhood: [ 'microhood' ], + disputed: [ 'disputed' ], + postalcode: [ 'postalcode' ], + ocean: [ 'ocean' ], + marinearea: [ 'marinearea' ] + }); + t.end(); + }); + + test('basic layer mapping', function(t) { + t.deepEquals(type_mapping.layer_mapping.venue, ['venue']); + t.deepEquals(type_mapping.layer_mapping.address, ['address']); + t.end(); + }); + + test('alias layer mapping', function(t) { + t.deepEquals(type_mapping.layer_mapping.coarse, + [ 'continent', 'empire', 'country', 'dependency', 'macroregion', + 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', + 'borough', 'neighbourhood', 'microhood', 'disputed', 'postalcode', + 'continent', 'ocean', 'marinearea']); + t.end(); + }); + + test('complete layers by source', function(t) { + t.deepEquals(type_mapping.layers_by_source, { + openstreetmap: [ 'address', 'venue', 'street' ], + openaddresses: [ 'address' ], + geonames: [ 'country', 'macroregion', 'region', 'county', 'localadmin', + 'locality', 'borough', 'neighbourhood', 'venue' ], + whosonfirst: [ 'continent', 'empire', 'country', 'dependency', 'macroregion', + 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', + 'borough', 'neighbourhood', 'microhood', 'disputed', 'venue', 'postalcode', + 'continent', 'ocean', 'marinearea' ] + }); + t.end(); + }); }; module.exports.all = function (tape, common) { diff --git a/test/unit/middleware/interpolate.js b/test/unit/middleware/interpolate.js index 3602592c..537c7444 100644 --- a/test/unit/middleware/interpolate.js +++ b/test/unit/middleware/interpolate.js @@ -277,6 +277,81 @@ module.exports.tests.success_conditions = (test, common) => { }); + test('results should be sorted first by address/non-address. previous ordering should otherwise be maintained via a stable sort', t => { + const service = (req, res, callback) => { + // results 5 and 7 will have interpolated results returned + // this is to ensure results are re-sorted to move the addresses first + if (res.id === 5 || res.id === 7) { + callback(null, { + properties: { + number: 17, + source: 'Source Abbr 1', + source_id: 'source 1 source id', + lat: 12.121212, + lon: 21.212121 + } + }); + } else { + // return empty results in most cases + callback(null, {}); + } + }; + + const logger = require('pelias-mock-logger')(); + + const controller = proxyquire('../../../middleware/interpolate', { + 'pelias-logger': logger + })(service, () => true); + + const req = { + clean: { + parsed_text: 'this is req.clean.parsed_text' + } + }; + const res = {}; + + // helper method to generate test results which default to streets + function generateTestStreets(id) { + return { + id: id+1, + layer: 'street', + name: { default: `name ${id+1}` }, + address_parts: {}, + source_id: 'original source_id' + }; + } + + // generate a set of street results of desired size + // NOTE: this set must be of 11 elements or greater + // Node.js uses stable insertion sort for arrays of 10 or fewer elements, + // but _unstable_ QuickSort for larger arrays + const resultCount = 11; + const sequence_array = Array.from(new Array(resultCount),(val,index)=>index); + res.data = sequence_array.map(generateTestStreets); + + controller(req, res, () => { + t.notOk(logger.hasErrorMessages(), 'there shouldn\'t be any error messages'); + + const results = res.data; + + t.equals(results.length, results.length, 'correct number of results should be returned'); + t.equals(results[0].layer, 'address', 'first result should be interpolated address'); + t.equals(results[1].layer, 'address', 'second result should be interpolated address'); + + // iterate through all remaining records, ensuring their ids are increasing, + // as was the case when the set of streets was originally generated + let previous_id; + for (let i = 2; i < results.length; i++) { + if (previous_id) { + t.ok(results[i].id > previous_id, `id ${results[i].id} should be higher than ${previous_id}, to ensure sort is stable`); + } + previous_id = results[i].id; + } + + t.end(); + }); + }); + test('service call returning error should not map in interpolated results for non-errors', t => { const service = (req, res, callback) => { if (res.id === 1) { diff --git a/test/unit/run.js b/test/unit/run.js index 52b6e519..f196dc53 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -29,13 +29,16 @@ var tests = [ require('./controller/predicates/is_admin_only_analysis'), require('./controller/predicates/is_coarse_reverse'), require('./controller/predicates/is_only_non_admin_layers'), + require('./controller/predicates/is_request_sources_includes_whosonfirst'), require('./controller/predicates/is_request_sources_only_whosonfirst'), + require('./controller/predicates/is_request_sources_undefined'), require('./helper/debug'), require('./helper/diffPlaces'), require('./helper/fieldValue'), require('./helper/geojsonify_place_details'), require('./helper/geojsonify'), require('./helper/logging'), + require('./helper/TypeMapping'), require('./helper/type_mapping'), require('./helper/stackTraceLine'), require('./middleware/access_log'), @@ -89,7 +92,6 @@ var tests = [ require('./sanitizer/_text'), require('./sanitizer/_text_addressit'), require('./sanitizer/_tokenizer'), - require('./sanitizer/_deprecate_quattroshapes'), require('./sanitizer/_categories'), require('./sanitizer/nearby'), require('./sanitizer/autocomplete'), diff --git a/test/unit/sanitizer/_deprecate_quattroshapes.js b/test/unit/sanitizer/_deprecate_quattroshapes.js deleted file mode 100644 index 6cbe1f09..00000000 --- a/test/unit/sanitizer/_deprecate_quattroshapes.js +++ /dev/null @@ -1,65 +0,0 @@ -var sanitizer = require('../../../sanitizer/_deprecate_quattroshapes')(); - -module.exports.tests = {}; - -module.exports.tests.warning_message_1 = function(test, common) { - test('[qs] should emit a deprecation warning', function(t) { - var raw = { sources: 'qs' }; - var clean = {}; - - var messages = sanitizer.sanitize(raw, clean); - t.deepEquals(messages, { - errors: [], - warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] - }, 'warning emitted'); - - t.end(); - }); -}; - -module.exports.tests.warning_message_2 = function(test, common) { - test('[quattroshapes] should emit a deprecation warning', function(t) { - var raw = { sources: 'quattroshapes' }; - var clean = {}; - - var messages = sanitizer.sanitize(raw, clean); - t.deepEquals(messages, { - errors: [], - warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] - }, 'warning emitted'); - - t.end(); - }); -}; - -module.exports.tests.rewrite = function(test, common) { - test('should rewrite qs and quattroshapes to whosonfirst', function(t) { - var raw = { sources: 'qs,quattroshapes,qs,quattroshapes,osm' }; - var clean = {}; - - sanitizer.sanitize(raw, clean); - t.equals(raw.sources,'osm,whosonfirst','use wof instead of qs'); - - t.end(); - }); -}; - -module.exports.all = function (tape, common) { - function test(name, testFunction) { - return tape('SANITIZE _deprecate_quattroshapes ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; diff --git a/test/unit/sanitizer/_text.js b/test/unit/sanitizer/_text.js index 1fde352b..6a368b41 100644 --- a/test/unit/sanitizer/_text.js +++ b/test/unit/sanitizer/_text.js @@ -37,7 +37,7 @@ module.exports.tests.text_parser = function(test, common) { const messages = sanitizer.sanitize(raw, clean); t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors'); + t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0']); t.deepEquals(messages.warnings, [], 'no warnings'); }); @@ -46,12 +46,98 @@ module.exports.tests.text_parser = function(test, common) { }); + test('should trim whitespace', t => { + var clean = {}; + var raw = { text: ` test \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim double quotes', t => { + var clean = {}; + var raw = { text: ` "test" \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim single quotes', t => { + var clean = {}; + var raw = { text: ` 'test' \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim German quotes', t => { + var clean = {}; + var raw = { text: ` „test“ \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim guillemets', t => { + var clean = {}; + var raw = { text: ` »test« \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim Chinese quotes', t => { + var clean = {}; + var raw = { text: ` ﹁「test」﹂ \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + test('return an array of expected parameters in object form for validation', (t) => { const expected = [{ name: 'text' }]; const validParameters = sanitizer.expected(); t.deepEquals(validParameters, expected); t.end(); }); + + test('whitespace-only input counts as empty', (t) => { + const raw = { text: ' ' }; + const clean = {}; + + const expected_clean = {}; + + const messages = sanitizer.sanitize(raw, clean); + + t.deepEquals(clean, expected_clean); + t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0']); + t.deepEquals(messages.warnings, [], 'no warnings'); + t.end(); + }); }; module.exports.all = (tape, common) => { diff --git a/test/unit/sanitizer/nearby.js b/test/unit/sanitizer/nearby.js index cff81b71..b4462943 100644 --- a/test/unit/sanitizer/nearby.js +++ b/test/unit/sanitizer/nearby.js @@ -26,14 +26,6 @@ module.exports.tests.sanitize = function(test, common) { } }; }, - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_targets': function (type) { if (['layers', 'sources'].indexOf(type) !== -1) { return { @@ -123,7 +115,6 @@ module.exports.tests.sanitize = function(test, common) { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_targets/layers', '_targets/sources', '_sources_and_layers', diff --git a/test/unit/sanitizer/reverse.js b/test/unit/sanitizer/reverse.js index c979ec67..22ac08aa 100644 --- a/test/unit/sanitizer/reverse.js +++ b/test/unit/sanitizer/reverse.js @@ -26,14 +26,6 @@ module.exports.tests.sanitize = function(test, common) { } }; }, - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_targets': function (type) { if (['layers', 'sources'].indexOf(type) !== -1) { return { @@ -115,7 +107,6 @@ module.exports.tests.sanitize = function(test, common) { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_targets/layers', '_targets/sources', '_sources_and_layers', diff --git a/test/unit/sanitizer/search.js b/test/unit/sanitizer/search.js index 9ac585ad..e72a6ccd 100644 --- a/test/unit/sanitizer/search.js +++ b/test/unit/sanitizer/search.js @@ -14,14 +14,6 @@ module.exports.tests.sanitize = (test, common) => { // the object contains a key called {function} sanitize, // which pushes the name of the sanitizer to {array} called_sanitizers const search = proxyquire('../../../sanitizer/search', { - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_single_scalar_parameters': function () { return { sanitize: () => { @@ -148,7 +140,6 @@ module.exports.tests.sanitize = (test, common) => { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_text', '_size', '_targets/layers', diff --git a/test/unit/sanitizer/structured_geocoding.js b/test/unit/sanitizer/structured_geocoding.js index 963aa21d..bff07b22 100644 --- a/test/unit/sanitizer/structured_geocoding.js +++ b/test/unit/sanitizer/structured_geocoding.js @@ -10,14 +10,6 @@ module.exports.tests.sanitize = function(test, common) { // rather than re-verify the functionality of all the sanitizers, this test just verifies that they // were all called correctly var search = proxyquire('../../../sanitizer/structured_geocoding', { - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_single_scalar_parameters': function () { return { sanitize: () => { @@ -151,7 +143,6 @@ module.exports.tests.sanitize = function(test, common) { var expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_synthesize_analysis', '_iso2_to_iso3', '_city_name_standardizer',