diff --git a/.circleci/docker.sh b/.circleci/docker.sh index 1acd51cb..0dfa433c 100644 --- a/.circleci/docker.sh +++ b/.circleci/docker.sh @@ -6,22 +6,19 @@ DATE=`date +%Y-%m-%d` DOCKER_REPOSITORY="pelias" DOCKER_PROJECT="${DOCKER_REPOSITORY}/${CIRCLE_PROJECT_REPONAME}" -# skip builds on greenkeeper branches -if [[ -z "${CIRCLE_BRANCH##*greenkeeper*}" ]]; then - exit 0 -fi +BRANCH="$(echo $CIRCLE_BRANCH | tr '/' '-')" #slashes are not valid in docker tags. replace with dashes # the name of the image that represents the "branch", that is an image that will be updated over time with the git branch # the production branch is changed to "latest", otherwise the git branch becomes the name of the version -if [[ "${CIRCLE_BRANCH}" == "production" ]]; then +if [[ "${BRANCH}" == "production" ]]; then DOCKER_BRANCH_IMAGE_VERSION="latest" else - DOCKER_BRANCH_IMAGE_VERSION="${CIRCLE_BRANCH}" + DOCKER_BRANCH_IMAGE_VERSION="$BRANCH" fi DOCKER_BRANCH_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_BRANCH_IMAGE_VERSION}" # the name of the image that represents the "tag", that is an image that is named with the date and git commit and will never be changed -DOCKER_TAG_IMAGE_VERSION="${CIRCLE_BRANCH}-${DATE}-${CIRCLE_SHA1}" +DOCKER_TAG_IMAGE_VERSION="${BRANCH}-${DATE}-${CIRCLE_SHA1}" DOCKER_TAG_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_TAG_IMAGE_VERSION}" # build image and login to docker hub diff --git a/README.md b/README.md index c3287227..d90f834c 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ The API recognizes the following properties under the top-level `api` key in you |`services`|*no*||service definitions for [point-in-polygon](https://github.com/pelias/pip-service), [libpostal](https://github.com/whosonfirst/go-whosonfirst-libpostal), [placeholder](https://github.com/pelias/placeholder), and [interpolation](https://github.com/pelias/interpolation) services. If missing (which is not recommended), the services will not be called.| |`defaultParameters.focus.point.lon`
`defaultParameters.focus.point.lat`|no | |default coordinates for focus point |`targets.layers_by_source`
`targets.source_aliases`
`targets.layer_aliases`|no | |custom values for which `sources` and `layers` the API accepts ([more info](https://github.com/pelias/api/pull/1131)). +|`attributionURL`|no| (autodetedted)|The full URL to use for the attribution link returned in all Pelias responses. Pelias will attempt to autodetect this host, but it will often be correct if, for example, there is a proxy between Pelias and its users. This parameter allows setting a specific URL to avoid any such issues| A good starting configuration file includes this section (fill in the service and Elasticsearch hosts as needed): diff --git a/controller/predicates/is_request_sources_includes_whosonfirst.js b/controller/predicates/is_request_sources_includes_whosonfirst.js new file mode 100644 index 00000000..fe3fcd8b --- /dev/null +++ b/controller/predicates/is_request_sources_includes_whosonfirst.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); +const Debug = require('../../helper/debug'); +const debugLog = new Debug('controller:predicates:is_request_sources_includes_whosonfirst'); +const stackTraceLine = require('../../helper/stackTraceLine'); + +// returns true IFF 'whosonfirst' is included in the requested sources +module.exports = (req, res) => { + const is_request_sources_includes_whosonfirst = _.get(req, 'clean.sources', []).includes( + 'whosonfirst' + ); + debugLog.push(req, () => ({ + reply: is_request_sources_includes_whosonfirst, + stack_trace: stackTraceLine() + })); + return is_request_sources_includes_whosonfirst; +}; diff --git a/controller/predicates/is_request_sources_undefined.js b/controller/predicates/is_request_sources_undefined.js new file mode 100644 index 00000000..77e8afea --- /dev/null +++ b/controller/predicates/is_request_sources_undefined.js @@ -0,0 +1,16 @@ +const _ = require('lodash'); +const Debug = require('../../helper/debug'); +const debugLog = new Debug('controller:predicates:is_request_sources_undefined'); +const stackTraceLine = require('../../helper/stackTraceLine'); + +// returns true IFF there are no requested sources +module.exports = (req, res) => { + const is_request_sources_undefined = _.isEmpty( + _.get(req, 'clean.sources') + ); + debugLog.push(req, () => ({ + reply: is_request_sources_undefined, + stack_trace: stackTraceLine() + })); + return is_request_sources_undefined; +}; diff --git a/package.json b/package.json index 2f361801..9c6e977e 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "homepage": "https://github.com/pelias/api", "license": "MIT", "main": "index.js", + "bin": "./bin/start", "scripts": { "audit": "npm shrinkwrap; node node_modules/nsp/bin/nsp check; rm npm-shrinkwrap.json;", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", @@ -50,16 +51,16 @@ "joi": "^13.1.3", "locale": "^0.1.0", "lodash": "^4.17.4", - "markdown": "0.5.0", + "markdown": "^0.5.0", "morgan": "^1.8.2", - "pelias-categories": "1.2.0", - "pelias-config": "3.0.2", - "pelias-labels": "1.8.0", - "pelias-logger": "0.4.2", - "pelias-microservice-wrapper": "1.4.0", - "pelias-model": "5.5.2", - "pelias-query": "9.1.1", - "pelias-sorting": "1.2.0", + "pelias-categories": "^1.2.0", + "pelias-config": "^3.0.2", + "pelias-labels": "^1.8.0", + "pelias-logger": "^0.4.2", + "pelias-microservice-wrapper": "^1.4.0", + "pelias-model": "^5.5.2", + "pelias-query": "^9.1.1", + "pelias-sorting": "^1.2.0", "predicates": "^2.0.0", "retry": "^0.12.0", "stable": "^0.1.8", @@ -72,14 +73,14 @@ "istanbul": "^0.4.2", "jshint": "^2.5.6", "nsp": "^3.0.0", - "pelias-mock-logger": "1.3.0", + "pelias-mock-logger": "^1.3.0", "precommit-hook": "^3.0.0", "proxyquire": "^2.0.0", "semantic-release": "^15.1.4", "source-map": "^0.7.0", - "tap-dot": "1.0.5", + "tap-dot": "^2.0.0", "tape": "^4.5.1", - "tmp": "0.0.33", + "tmp": "^0.0.33", "uglify-js": "^3.0.4" }, "pre-commit": [ diff --git a/public/attribution.md b/public/attribution.md index f2b34bd3..fc96eaa8 100644 --- a/public/attribution.md +++ b/public/attribution.md @@ -1,7 +1,7 @@ ## Attribution * Geocoding by [Pelias](https://pelias.io). * Data from - * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/) - * [OpenAddresses](http://openaddresses.io) under a [Creative Commons Zero](https://github.com/openaddresses/openaddresses/blob/master/sources/LICENSE) public domain designation - * [GeoNames](http://www.geonames.org/) under [CC-BY-3.0](https://creativecommons.org/licenses/by/2.0/) - * [WhosOnFirst](https://www.whosonfirst.org/) under [various licenses](https://github.com/whosonfirst/whosonfirst-data/blob/master/LICENSE.md) + * [OpenStreetMap](http://www.openstreetmap.org/copyright) © OpenStreetMap contributors under [ODbL](http://opendatacommons.org/licenses/odbl/). Also see the [OSM Geocoding Guidelines](https://wiki.osmfoundation.org/wiki/Licence/Community_Guidelines/Geocoding_-_Guideline) for acceptable use. + * [OpenAddresses](http://openaddresses.io) under [various public-domain and share-alike licenses](http://results.openaddresses.io/) + * [GeoNames](http://www.geonames.org/) under [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) + * [WhosOnFirst](https://www.whosonfirst.org/) under [various CC-BY or CC-0 equivalent licenses](https://whosonfirst.org/docs/licenses/) diff --git a/query/autocomplete.js b/query/autocomplete.js index 8d4cf257..f5b37982 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -7,7 +7,6 @@ const logger = require('pelias-logger').get('api'); // additional views (these may be merged in to pelias/query at a later date) var views = { ngrams_strict: require('./view/ngrams_strict'), - focus_selected_layers: require('./view/focus_selected_layers'), ngrams_last_token_only: require('./view/ngrams_last_token_only'), phrase_first_tokens_only: require('./view/phrase_first_tokens_only'), pop_subquery: require('./view/pop_subquery'), @@ -42,7 +41,7 @@ query.score( peliasQuery.view.admin('neighbourhood') ); // scoring boost query.score( views.boost_exact_matches ); -query.score( views.focus_selected_layers( views.ngrams_strict ) ); +query.score( peliasQuery.view.focus( views.ngrams_strict ) ); query.score( peliasQuery.view.popularity( views.pop_subquery ) ); query.score( peliasQuery.view.population( views.pop_subquery ) ); diff --git a/query/view/focus_selected_layers.js b/query/view/focus_selected_layers.js deleted file mode 100644 index 4fda1780..00000000 --- a/query/view/focus_selected_layers.js +++ /dev/null @@ -1,38 +0,0 @@ -var peliasQuery = require('pelias-query'); - -/** - This view is the same as `peliasQuery.view.focus` with one exception: - - if the view is generated successfully, we add a 'filter' clause which - restricts the targeted '_type' to be in the list specified below. - - documents which are not in the '_type' list below will simply score 0 for - this section of the query. -**/ - -module.exports = function( subview ){ - return function( vs ){ - - // don't perform this query on single character inputs - // as its too unperformant to sort a large part of the index. - if( vs.var('input:name').get().length < 2 ){ - return null; - } - - if( !subview ){ return null; } // subview validation failed - var macroView = peliasQuery.view.focus( subview ); - if( !macroView ){ return null; } // macroView validation failed - var view = macroView( vs ); - - if( view && view.hasOwnProperty('function_score') ){ - view.function_score.filter = { - 'or': [ - { 'term': { 'layer': 'venue' } }, - { 'term': { 'layer': 'address' } } - ] - }; - } - - return view; - }; -}; diff --git a/routes/v1.js b/routes/v1.js index 6b40f1d6..79243794 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -80,6 +80,9 @@ const hasRequestCategories = require('../controller/predicates/has_request_param const isOnlyNonAdminLayers = require('../controller/predicates/is_only_non_admin_layers'); // this can probably be more generalized const isRequestSourcesOnlyWhosOnFirst = require('../controller/predicates/is_request_sources_only_whosonfirst'); +const isRequestSourcesIncludesWhosOnFirst = require('../controller/predicates/is_request_sources_includes_whosonfirst'); +const isRequestSourcesUndefined = require('../controller/predicates/is_request_sources_undefined'); + const hasRequestParameter = require('../controller/predicates/has_request_parameter'); const hasParsedTextProperties = require('../controller/predicates/has_parsed_text_properties'); @@ -167,9 +170,14 @@ function addRoutes(app, peliasConfig) { ) ), any( - // only geodisambiguate if libpostal returned only admin areas or libpostal was skipped - isAdminOnlyAnalysis, - isRequestSourcesOnlyWhosOnFirst + isRequestSourcesOnlyWhosOnFirst, + all( + isAdminOnlyAnalysis, + any( + isRequestSourcesUndefined, + isRequestSourcesIncludesWhosOnFirst + ) + ) ) ); @@ -259,7 +267,6 @@ function addRoutes(app, peliasConfig) { // helpers to replace vague booleans const geometricFiltersApply = true; - const geometricFiltersDontApply = false; var base = '/v1/'; @@ -278,7 +285,7 @@ function addRoutes(app, peliasConfig) { middleware.calcSize(), controllers.libpostal(libpostalService, libpostalShouldExecute), controllers.placeholder(placeholderService, geometricFiltersApply, placeholderGeodisambiguationShouldExecute), - controllers.placeholder(placeholderService, geometricFiltersDontApply, placeholderIdsLookupShouldExecute), + controllers.placeholder(placeholderService, geometricFiltersApply, placeholderIdsLookupShouldExecute), controllers.search_with_ids(peliasConfig.api, esclient, queries.address_using_ids, searchWithIdsShouldExecute), // 3rd parameter is which query module to use, use fallback first, then // use original search strategy if first query didn't return anything diff --git a/sanitizer/_deprecate_quattroshapes.js b/sanitizer/_deprecate_quattroshapes.js deleted file mode 100644 index 1db2fef1..00000000 --- a/sanitizer/_deprecate_quattroshapes.js +++ /dev/null @@ -1,43 +0,0 @@ -var _ = require('lodash'); - -/** - In the process of phasing out the 'quattroshapes' source in favour of 'whosonfirst' - we will emit a warning to users so they can begin upgrading their clients. - - In the interim we will automatically rewrite all requests for quattroshapes to whosonfirst. - - @todo: this is only temporary - @see: https://github.com/pelias/api/issues/442 -**/ - -function _sanitize( raw, clean, opts ) { - // error & warning messages - var messages = { errors: [], warnings: [] }; - - // only applicably when 'sources' param is privided - if( raw.hasOwnProperty('sources') ){ - - var sources = raw.sources.split(','); - if (_.includes(sources, 'quattroshapes') || _.includes(sources, 'qs')) { - - // emit a warning message so users can transition. - messages.warnings.push('You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'); - - // user requested 'quattroshapes', we will give them 'whosonfirst' instead. - sources = _.without(sources, 'quattroshapes', 'qs'); - sources.push('whosonfirst'); - raw.sources = sources.join(','); - } - } - - return messages; -} - -module.exports = () => ({ - sanitize: _sanitize -}); diff --git a/sanitizer/_text.js b/sanitizer/_text.js index 1df63411..bae2d4a1 100644 --- a/sanitizer/_text.js +++ b/sanitizer/_text.js @@ -1,6 +1,9 @@ const check = require('check-types'); const _ = require('lodash'); +// ref: https://en.wikipedia.org/wiki/Quotation_mark +const QUOTES = `"'«»‘’‚‛“”„‟‹›⹂「」『』〝〞〟﹁﹂﹃﹄"'「」`; + // validate texts, convert types and apply defaults function _sanitize( raw, clean ){ @@ -10,12 +13,12 @@ function _sanitize( raw, clean ){ // invalid input 'text' // must call `!check.nonEmptyString` since `check.emptyString` returns // `false` for `undefined` and `null` - if( !check.nonEmptyString( raw.text ) ){ - messages.errors.push('invalid param \'text\': text length, must be >0'); + const text = _.trim( _.trim( raw.text ), QUOTES ); + if( !check.nonEmptyString( text ) ){ + messages.errors.push('invalid param \'text\': text length, must be >0'); } else { - clean.text = raw.text; - + clean.text = text; } return messages; diff --git a/sanitizer/nearby.js b/sanitizer/nearby.js index 1cd3b276..087492de 100644 --- a/sanitizer/nearby.js +++ b/sanitizer/nearby.js @@ -5,7 +5,6 @@ var type_mapping = require('../helper/type_mapping'); var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitizers, must be run after them diff --git a/sanitizer/reverse.js b/sanitizer/reverse.js index ce14e862..c3bb89dc 100644 --- a/sanitizer/reverse.js +++ b/sanitizer/reverse.js @@ -3,7 +3,6 @@ var sanitizeAll = require('../sanitizer/sanitizeAll'), sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping), // depends on the layers and sources sanitizers, must be run after them diff --git a/sanitizer/search.js b/sanitizer/search.js index 008ae693..4b74bd2d 100644 --- a/sanitizer/search.js +++ b/sanitizer/search.js @@ -5,7 +5,6 @@ module.exports.middleware = (_api_pelias_config) => { var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), text: require('../sanitizer/_text')(), size: require('../sanitizer/_size')(/* use defaults*/), layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping), diff --git a/sanitizer/structured_geocoding.js b/sanitizer/structured_geocoding.js index 670a26aa..047bb8ff 100644 --- a/sanitizer/structured_geocoding.js +++ b/sanitizer/structured_geocoding.js @@ -6,7 +6,6 @@ module.exports.middleware = (_api_pelias_config) => { var sanitizers = { singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(), debug: require('../sanitizer/_debug')(), - quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(), synthesize_analysis: require('../sanitizer/_synthesize_analysis')(), iso2_to_iso3: require('../sanitizer/_iso2_to_iso3')(), city_name_standardizer: require('../sanitizer/_city_name_standardizer')(), diff --git a/test/ciao/reverse/sources_deprecation_warning.coffee b/test/ciao/reverse/sources_deprecation_warning.coffee deleted file mode 100644 index 51445800..00000000 --- a/test/ciao/reverse/sources_deprecation_warning.coffee +++ /dev/null @@ -1,34 +0,0 @@ - -#> quattroshapes is being phased out and so should emit a warning message -path: '/v1/reverse?point.lat=1&point.lon=2&sources=qs' - -#? 200 ok -response.statusCode.should.be.equal 200 -response.should.have.header 'charset', 'utf8' -response.should.have.header 'content-type', 'application/json; charset=utf-8' - -#? valid geocoding block -should.exist json.geocoding -should.exist json.geocoding.version -should.exist json.geocoding.attribution -should.exist json.geocoding.query -should.exist json.geocoding.engine -should.exist json.geocoding.engine.name -should.exist json.geocoding.engine.author -should.exist json.geocoding.engine.version -should.exist json.geocoding.timestamp - -#? valid geojson -json.type.should.be.equal 'FeatureCollection' -json.features.should.be.instanceof Array - -#? expected errors -should.not.exist json.geocoding.errors - -#? expected warnings -should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] - -#? inputs -json.geocoding.query['size'].should.eql 10 -json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/ciao/search/sources_deprecation_warning.coffee b/test/ciao/search/sources_deprecation_warning.coffee deleted file mode 100644 index fe82d298..00000000 --- a/test/ciao/search/sources_deprecation_warning.coffee +++ /dev/null @@ -1,35 +0,0 @@ - -#> quattroshapes is being phased out and so should emit a warning message -path: '/v1/search?sources=qs&text=a' - -#? 200 ok -response.statusCode.should.be.equal 200 -response.should.have.header 'charset', 'utf8' -response.should.have.header 'content-type', 'application/json; charset=utf-8' - -#? valid geocoding block -should.exist json.geocoding -should.exist json.geocoding.version -should.exist json.geocoding.attribution -should.exist json.geocoding.query -should.exist json.geocoding.engine -should.exist json.geocoding.engine.name -should.exist json.geocoding.engine.author -should.exist json.geocoding.engine.version -should.exist json.geocoding.timestamp - -#? valid geojson -json.type.should.be.equal 'FeatureCollection' -json.features.should.be.instanceof Array - -#? expected errors -should.not.exist json.geocoding.errors - -#? expected warnings -should.exist json.geocoding.warnings -json.geocoding.warnings.should.eql ['You are using Quattroshapes as a data source in this query. Quattroshapes has been disabled as a data source for Mapzen Search, and has beenreplaced by Who\'s on First, an actively maintained data project based on QuattroshapesYour existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as `sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.' ] - -#? inputs -json.geocoding.query['size'].should.eql 10 -json.geocoding.query['text'].should.eql 'a' -json.geocoding.query.sources.should.eql ['whosonfirst'] # should use 'whosonfirst' instead of 'quattroshapes' diff --git a/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js b/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js new file mode 100644 index 00000000..97c0d95f --- /dev/null +++ b/test/unit/controller/predicates/is_request_sources_includes_whosonfirst.js @@ -0,0 +1,90 @@ +const _ = require('lodash'); +const is_request_sources_includes_whosonfirst = require('../../../../controller/predicates/is_request_sources_includes_whosonfirst'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + t.ok(_.isFunction(is_request_sources_includes_whosonfirst), 'is_request_sources_includes_whosonfirst is a function'); + t.end(); + }); +}; + +module.exports.tests.true_conditions = (test, common) => { + test('sources includes \'whosonfirst\' should return true', (t) => { + const req = { + clean: { + sources: [ + 'whosonfirst', + 'not whosonfirst' + ] + } + }; + + t.ok(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('empty req.clean.sources should return false', (t) => { + const req = { + clean: { + sources: [] + } + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + +}; + +module.exports.tests.false_conditions = (test, common) => { + test('undefined req should return false', (t) => { + t.notOk(is_request_sources_includes_whosonfirst(undefined)); + t.end(); + + }); + + test('undefined req.clean should return false', (t) => { + const req = {}; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('undefined req.clean.sources should return false', (t) => { + const req = { + clean: {} + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + + }); + + test('sources not \'whosonfirst\' should return false', (t) => { + const req = { + clean: { + sources: [ + 'not whosonfirst' + ] + } + }; + + t.notOk(is_request_sources_includes_whosonfirst(req)); + t.end(); + }); +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`GET /is_request_sources_includes_whosonfirst ${name}`, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/controller/predicates/is_request_sources_undefined.js b/test/unit/controller/predicates/is_request_sources_undefined.js new file mode 100644 index 00000000..d7486bb8 --- /dev/null +++ b/test/unit/controller/predicates/is_request_sources_undefined.js @@ -0,0 +1,78 @@ +const _ = require('lodash'); +const is_request_sources_undefined = require('../../../../controller/predicates/is_request_sources_undefined'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + t.ok(_.isFunction(is_request_sources_undefined), 'is_request_sources_undefined is a function'); + t.end(); + }); +}; + +module.exports.tests.true_conditions = (test, common) => { + test('undefined req should return true', (t) => { + + t.ok(is_request_sources_undefined(undefined)); + t.end(); + + }); + + test('undefined req.clean should return true', (t) => { + const req = {}; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + + test('undefined req.clean.sources should return true', (t) => { + const req = { + clean: {} + }; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + + test('empty req.clean.sources should return true', (t) => { + const req = { + clean: { + sources: [] + } + }; + + t.ok(is_request_sources_undefined(req)); + t.end(); + + }); + +}; + +module.exports.tests.false_conditions = (test, common) => { + test('sources not empty should return false', (t) => { + const req = { + clean: { + sources: [ + 'not empty' + ] + } + }; + + t.notOk(is_request_sources_undefined(req)); + t.end(); + + }); + +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`GET /is_request_sources_undefined ${name}`, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/fixture/autocomplete_linguistic_focus.js b/test/unit/fixture/autocomplete_linguistic_focus.js index 219bea36..2acc0e1d 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus.js +++ b/test/unit/fixture/autocomplete_linguistic_focus.js @@ -46,21 +46,7 @@ module.exports = { 'weight': 15 }], 'score_mode': 'avg', - 'boost_mode': 'replace', - 'filter': { - 'or': [ - { - 'term': { - 'layer': 'venue' - } - }, - { - 'term': { - 'layer': 'address' - } - } - ] - } + 'boost_mode': 'replace' } },{ 'function_score': { diff --git a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js index 80e44f81..d3029455 100644 --- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js +++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js @@ -46,21 +46,7 @@ module.exports = { 'weight': 15 }], 'score_mode': 'avg', - 'boost_mode': 'replace', - 'filter': { - 'or': [ - { - 'term': { - 'layer': 'venue' - } - }, - { - 'term': { - 'layer': 'address' - } - } - ] - } + 'boost_mode': 'replace' } },{ 'function_score': { diff --git a/test/unit/run.js b/test/unit/run.js index 404cd86b..f196dc53 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -29,7 +29,9 @@ var tests = [ require('./controller/predicates/is_admin_only_analysis'), require('./controller/predicates/is_coarse_reverse'), require('./controller/predicates/is_only_non_admin_layers'), + require('./controller/predicates/is_request_sources_includes_whosonfirst'), require('./controller/predicates/is_request_sources_only_whosonfirst'), + require('./controller/predicates/is_request_sources_undefined'), require('./helper/debug'), require('./helper/diffPlaces'), require('./helper/fieldValue'), @@ -90,7 +92,6 @@ var tests = [ require('./sanitizer/_text'), require('./sanitizer/_text_addressit'), require('./sanitizer/_tokenizer'), - require('./sanitizer/_deprecate_quattroshapes'), require('./sanitizer/_categories'), require('./sanitizer/nearby'), require('./sanitizer/autocomplete'), diff --git a/test/unit/sanitizer/_deprecate_quattroshapes.js b/test/unit/sanitizer/_deprecate_quattroshapes.js deleted file mode 100644 index 6cbe1f09..00000000 --- a/test/unit/sanitizer/_deprecate_quattroshapes.js +++ /dev/null @@ -1,65 +0,0 @@ -var sanitizer = require('../../../sanitizer/_deprecate_quattroshapes')(); - -module.exports.tests = {}; - -module.exports.tests.warning_message_1 = function(test, common) { - test('[qs] should emit a deprecation warning', function(t) { - var raw = { sources: 'qs' }; - var clean = {}; - - var messages = sanitizer.sanitize(raw, clean); - t.deepEquals(messages, { - errors: [], - warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] - }, 'warning emitted'); - - t.end(); - }); -}; - -module.exports.tests.warning_message_2 = function(test, common) { - test('[quattroshapes] should emit a deprecation warning', function(t) { - var raw = { sources: 'quattroshapes' }; - var clean = {}; - - var messages = sanitizer.sanitize(raw, clean); - t.deepEquals(messages, { - errors: [], - warnings: ['You are using Quattroshapes as a data source in this query. ' + - 'Quattroshapes has been disabled as a data source for Mapzen Search, and has been' + - 'replaced by Who\'s on First, an actively maintained data project based on Quattroshapes' + - 'Your existing queries WILL CONTINUE TO WORK for the foreseeable future, but results will ' + - 'be coming from Who\'s on First and `sources=quattroshapes` will be interpreted as ' + - '`sources=whosonfirst`. If you have any questions, please email pelias.team@gmail.com.'] - }, 'warning emitted'); - - t.end(); - }); -}; - -module.exports.tests.rewrite = function(test, common) { - test('should rewrite qs and quattroshapes to whosonfirst', function(t) { - var raw = { sources: 'qs,quattroshapes,qs,quattroshapes,osm' }; - var clean = {}; - - sanitizer.sanitize(raw, clean); - t.equals(raw.sources,'osm,whosonfirst','use wof instead of qs'); - - t.end(); - }); -}; - -module.exports.all = function (tape, common) { - function test(name, testFunction) { - return tape('SANITIZE _deprecate_quattroshapes ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; diff --git a/test/unit/sanitizer/_text.js b/test/unit/sanitizer/_text.js index 1fde352b..6a368b41 100644 --- a/test/unit/sanitizer/_text.js +++ b/test/unit/sanitizer/_text.js @@ -37,7 +37,7 @@ module.exports.tests.text_parser = function(test, common) { const messages = sanitizer.sanitize(raw, clean); t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors'); + t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0']); t.deepEquals(messages.warnings, [], 'no warnings'); }); @@ -46,12 +46,98 @@ module.exports.tests.text_parser = function(test, common) { }); + test('should trim whitespace', t => { + var clean = {}; + var raw = { text: ` test \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim double quotes', t => { + var clean = {}; + var raw = { text: ` "test" \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim single quotes', t => { + var clean = {}; + var raw = { text: ` 'test' \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim German quotes', t => { + var clean = {}; + var raw = { text: ` „test“ \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim guillemets', t => { + var clean = {}; + var raw = { text: ` »test« \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + + test('should trim Chinese quotes', t => { + var clean = {}; + var raw = { text: ` ﹁「test」﹂ \n ` }; + const messages = sanitizer.sanitize(raw, clean); + + t.equals(clean.text, 'test'); + t.deepEquals(messages.errors, [], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + t.end(); + }); + test('return an array of expected parameters in object form for validation', (t) => { const expected = [{ name: 'text' }]; const validParameters = sanitizer.expected(); t.deepEquals(validParameters, expected); t.end(); }); + + test('whitespace-only input counts as empty', (t) => { + const raw = { text: ' ' }; + const clean = {}; + + const expected_clean = {}; + + const messages = sanitizer.sanitize(raw, clean); + + t.deepEquals(clean, expected_clean); + t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0']); + t.deepEquals(messages.warnings, [], 'no warnings'); + t.end(); + }); }; module.exports.all = (tape, common) => { diff --git a/test/unit/sanitizer/nearby.js b/test/unit/sanitizer/nearby.js index cff81b71..b4462943 100644 --- a/test/unit/sanitizer/nearby.js +++ b/test/unit/sanitizer/nearby.js @@ -26,14 +26,6 @@ module.exports.tests.sanitize = function(test, common) { } }; }, - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_targets': function (type) { if (['layers', 'sources'].indexOf(type) !== -1) { return { @@ -123,7 +115,6 @@ module.exports.tests.sanitize = function(test, common) { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_targets/layers', '_targets/sources', '_sources_and_layers', diff --git a/test/unit/sanitizer/reverse.js b/test/unit/sanitizer/reverse.js index c979ec67..22ac08aa 100644 --- a/test/unit/sanitizer/reverse.js +++ b/test/unit/sanitizer/reverse.js @@ -26,14 +26,6 @@ module.exports.tests.sanitize = function(test, common) { } }; }, - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_targets': function (type) { if (['layers', 'sources'].indexOf(type) !== -1) { return { @@ -115,7 +107,6 @@ module.exports.tests.sanitize = function(test, common) { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_targets/layers', '_targets/sources', '_sources_and_layers', diff --git a/test/unit/sanitizer/search.js b/test/unit/sanitizer/search.js index 9ac585ad..e72a6ccd 100644 --- a/test/unit/sanitizer/search.js +++ b/test/unit/sanitizer/search.js @@ -14,14 +14,6 @@ module.exports.tests.sanitize = (test, common) => { // the object contains a key called {function} sanitize, // which pushes the name of the sanitizer to {array} called_sanitizers const search = proxyquire('../../../sanitizer/search', { - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_single_scalar_parameters': function () { return { sanitize: () => { @@ -148,7 +140,6 @@ module.exports.tests.sanitize = (test, common) => { const expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_text', '_size', '_targets/layers', diff --git a/test/unit/sanitizer/structured_geocoding.js b/test/unit/sanitizer/structured_geocoding.js index 963aa21d..bff07b22 100644 --- a/test/unit/sanitizer/structured_geocoding.js +++ b/test/unit/sanitizer/structured_geocoding.js @@ -10,14 +10,6 @@ module.exports.tests.sanitize = function(test, common) { // rather than re-verify the functionality of all the sanitizers, this test just verifies that they // were all called correctly var search = proxyquire('../../../sanitizer/structured_geocoding', { - '../sanitizer/_deprecate_quattroshapes': function () { - return { - sanitize: () => { - called_sanitizers.push('_deprecate_quattroshapes'); - return { errors: [], warnings: [] }; - } - }; - }, '../sanitizer/_single_scalar_parameters': function () { return { sanitize: () => { @@ -151,7 +143,6 @@ module.exports.tests.sanitize = function(test, common) { var expected_sanitizers = [ '_single_scalar_parameters', '_debug', - '_deprecate_quattroshapes', '_synthesize_analysis', '_iso2_to_iso3', '_city_name_standardizer',