pelias
/
api
mirror of https://github.com/pelias/api.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

call runAllChecks for search, search_fallback, autocomplete & structured sanitizing

pull/942/head
Lily He 7 years ago
parent
0486967526
commit
1f527be3e3
8 changed files with 548 additions and 209 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 20
      sanitizer/autocomplete.js
  2. 26
      sanitizer/search.js
  3. 34
      sanitizer/search_fallback.js
  4. 28
      sanitizer/structured_geocoding.js
  5. 79
      test/unit/sanitizer/autocomplete.js
  6. 88
      test/unit/sanitizer/search.js
  7. 197
      test/unit/sanitizer/search_fallback.js
  8. 71
      test/unit/sanitizer/structured_geocoding.js

20
sanitizer/autocomplete.js
Unescape View File

@ -4,25 +4,23 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
// middleware
module.exports.middleware = (_api_pelias_config) => {
var sanitizers = {
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
text: require('../sanitizer/_text_addressit'),
tokenizer: require('../sanitizer/_tokenizer'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
text: require('../sanitizer/_text_addressit')(),
tokenizer: require('../sanitizer/_tokenizer')(),
size: require('../sanitizer/_size')(10, 10, 10),
layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
// depends on the layers and sources sanitizers, must be run after them
sources_and_layers: require('../sanitizer/_sources_and_layers'),
sources_and_layers: require('../sanitizer/_sources_and_layers')(),
private: require('../sanitizer/_flag_bool')('private', false),
location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
geo_autocomplete: require('../sanitizer/_geo_autocomplete'),
boundary_country: require('../sanitizer/_boundary_country'),
categories: require('../sanitizer/_categories')
geo_autocomplete: require('../sanitizer/_geo_autocomplete')(),
boundary_country: require('../sanitizer/_boundary_country')(),
categories: require('../sanitizer/_categories')()
};
var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
return function( req, res, next ){
sanitize( req, function( err, clean ){
return ( req, res, next ) => {
sanitizeAll.runAllChecks(req, sanitizers, ( err, clean ) => {
if( err ){
res.status(400); // 400 Bad Request
return next(err);

26
sanitizer/search.js
Unescape View File

@ -3,27 +3,29 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
// middleware
module.exports.middleware = (_api_pelias_config) => {
var sanitizers = {
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
text: require('../sanitizer/_text'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
text: require('../sanitizer/_text')(),
size: require('../sanitizer/_size')(/* use defaults*/),
layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
// depends on the layers and sources sanitizers, must be run after them
sources_and_layers: require('../sanitizer/_sources_and_layers'),
sources_and_layers: require('../sanitizer/_sources_and_layers')(),
private: require('../sanitizer/_flag_bool')('private', false),
location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
geo_search: require('../sanitizer/_geo_search'),
boundary_country: require('../sanitizer/_boundary_country'),
categories: require('../sanitizer/_categories'),
geo_search: require('../sanitizer/_geo_search')(),
boundary_country: require('../sanitizer/_boundary_country')(),
categories: require('../sanitizer/_categories')(),
// this can go away once geonames has been abrogated
geonames_warnings: require('../sanitizer/_geonames_warnings')
geonames_warnings: require('../sanitizer/_geonames_warnings')()
};
var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
return function( req, res, next ){
sanitize( req, function( err, clean ){
return ( req, res, next ) => {
sanitizeAll.runAllChecks(req, sanitizers, ( err, clean ) => {
if( err ){
res.status(400); // 400 Bad Request
return next(err);
}
next();
});
};

34
sanitizer/search_fallback.js
Unescape View File

@ -0,0 +1,34 @@
var sanitizeAll = require('../sanitizer/sanitizeAll'),
sanitizers = {
text: require('../sanitizer/_text_addressit')()
};
var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' );
var _ = require('lodash');
// middleware
module.exports.middleware = function( req, res, next ){
// if res.data already has results then don't call the _text_autocomplete sanitizer
// this has been put into place for when the libpostal integration way of querying
// ES doesn't return anything and we want to fallback to the old logic
if (_.get(res, 'data', []).length > 0) {
return next();
}
// log the query that caused a fallback since libpostal+new-queries didn't return anything
if (req.path === '/v1/search') {
const queryText = logging.isDNT(req) ? '[text removed]' : req.clean.text;
logger.info(`fallback queryText: ${queryText}`);
}
// calls to sanitize the input
// omits check if parameters are valid since it only calls _text_addressit
sanitizeAll.sanitize(req, sanitizers, ( err, clean ) => {
if( err ){
res.status(400); // 400 Bad Request
return next(err);
}
next();
});
};

28
sanitizer/structured_geocoding.js
Unescape View File

@ -4,27 +4,31 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
// middleware
module.exports.middleware = (_api_pelias_config) => {
var sanitizers = {
singleScalarParameters: require('../sanitizer/_single_scalar_parameters'),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes'),
synthesize_analysis: require('../sanitizer/_synthesize_analysis'),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3'),
city_name_standardizer: require('../sanitizer/_city_name_standardizer'),
singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
quattroshapes_deprecation: require('../sanitizer/_deprecate_quattroshapes')(),
synthesize_analysis: require('../sanitizer/_synthesize_analysis')(),
iso2_to_iso3: require('../sanitizer/_iso2_to_iso3')(),
city_name_standardizer: require('../sanitizer/_city_name_standardizer')(),
size: require('../sanitizer/_size')(/* use defaults*/),
layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
// depends on the layers and sources sanitizers, must be run after them
sources_and_layers: require('../sanitizer/_sources_and_layers'),
sources_and_layers: require('../sanitizer/_sources_and_layers')(),
private: require('../sanitizer/_flag_bool')('private', false),
location_bias: require('../sanitizer/_location_bias')(_api_pelias_config.defaultParameters),
geo_search: require('../sanitizer/_geo_search'),
boundary_country: require('../sanitizer/_boundary_country'),
categories: require('../sanitizer/_categories')
geo_search: require('../sanitizer/_geo_search')(),
boundary_country: require('../sanitizer/_boundary_country')(),
categories: require('../sanitizer/_categories')()
};
var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
return function( req, res, next ){
sanitize( req, function( err, clean ){
return ( req, res, next ) => {
sanitizeAll.runAllChecks(req, sanitizers, ( err, clean ) => {
if( err ){
res.status(400); // 400 Bad Request
return next(err);
}
next();
});
};
};

79
test/unit/sanitizer/autocomplete.js
Unescape View File

@ -8,82 +8,111 @@ module.exports.tests.sanitizers = function(test, common) {
var called_sanitizers = [];
var autocomplete = proxyquire('../../../sanitizer/autocomplete', {
'../sanitizer/_single_scalar_parameters': () => {
'../sanitizer/_single_scalar_parameters': function () {
return {
sanitize: () => {
called_sanitizers.push('_single_scalar_parameters');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_text_addressit': () => {
'../sanitizer/_text_addressit': function () {
return {
sanitize: () => {
called_sanitizers.push('_text_addressit');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_tokenizer': () => {
'../sanitizer/_tokenizer': function () {
return {
sanitize: () => {
called_sanitizers.push('_tokenizer');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_size': function () {
if (_.isEqual(_.values(arguments), [10, 10, 10])) {
return () => {
return {
sanitize: () => {
called_sanitizers.push('_size');
return { errors: [], warnings: [] };
}
};
} else {
throw new Error('incorrect parameters passed to _size');
}
},
'../sanitizer/_targets': (type) => {
'../sanitizer/_targets': function (type) {
if (['layers', 'sources'].indexOf(type) !== -1) {
return () => {
return {
sanitize: () => {
called_sanitizers.push(`_targets/${type}`);
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _targets');
}
},
'../sanitizer/_sources_and_layers': () => {
'../sanitizer/_sources_and_layers': function () {
return {
sanitize: () => {
called_sanitizers.push('_sources_and_layers');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_flag_bool': function () {
if (arguments[0] === 'private' && arguments[1] === false) {
return () => {
return {
sanitize: () => {
called_sanitizers.push('_flag_bool');
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _flag_bool');
}
},
'../sanitizer/_location_bias': (defaultParameters) => {
'../sanitizer/_location_bias': function (defaultParameters) {
return {
sanitize: () => {
if (defaultParameters.key === 'value'){
return () => {
called_sanitizers.push('_location_bias');
return { errors: [], warnings: [] };
};
} else {
throw new Error('incorrect parameter passed to _location_bias');
}
}
};
},
'../sanitizer/_geo_autocomplete': () => {
'../sanitizer/_geo_autocomplete': function () {
return {
sanitize: () => {
called_sanitizers.push('_geo_autocomplete');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_boundary_country': () => {
'../sanitizer/_boundary_country': function () {
return {
sanitize: () => {
called_sanitizers.push('_boundary_country');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_categories': () => {
'../sanitizer/_categories': function () {
return {
sanitize: () => {
called_sanitizers.push('_categories');
return { errors: [], warnings: [] };
},
}
};
}
});
const expected_sanitizers = [

88
test/unit/sanitizer/search.js
Unescape View File

@ -9,87 +9,123 @@ module.exports.tests.sanitize = (test, common) => {
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
// each mock dependency is a function that returns an object.
// the object contains a key called {function} sanitize,
// which pushes the name of the sanitizer to {array} called_sanitizers
const search = proxyquire('../../../sanitizer/search', {
'../sanitizer/_deprecate_quattroshapes': () => {
'../sanitizer/_deprecate_quattroshapes': function () {
return {
sanitize: () => {
called_sanitizers.push('_deprecate_quattroshapes');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_single_scalar_parameters': () => {
'../sanitizer/_single_scalar_parameters': function () {
return {
sanitize: () => {
called_sanitizers.push('_single_scalar_parameters');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_text': () => {
'../sanitizer/_text': function () {
return {
sanitize: () => {
called_sanitizers.push('_text');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_size': function() {
if (_.isEmpty(arguments)) {
return () => {
return {
sanitize: () => {
called_sanitizers.push('_size');
return { errors: [], warnings: [] };
}
};
} else {
throw new Error('should not have passed any parameters to _size');
}
},
'../sanitizer/_targets': (type) => {
'../sanitizer/_targets': function (type) {
if (['layers', 'sources'].indexOf(type) !== -1) {
return () => {
return {
sanitize: () => {
called_sanitizers.push(`_targets/${type}`);
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _targets');
}
},
'../sanitizer/_sources_and_layers': () => {
'../sanitizer/_sources_and_layers': function () {
return {
sanitize: () => {
called_sanitizers.push('_sources_and_layers');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_flag_bool': function () {
if (arguments[0] === 'private' && arguments[1] === false) {
return () => {
return {
sanitize: () => {
called_sanitizers.push('_flag_bool');
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _flag_bool');
}
},
'../sanitizer/_geo_search': () => {
'../sanitizer/_geo_search': function () {
return {
sanitize: () => {
called_sanitizers.push('_geo_search');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_boundary_country': () => {
'../sanitizer/_boundary_country': function () {
return {
sanitize: () => {
called_sanitizers.push('_boundary_country');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_categories': () => {
'../sanitizer/_categories': function () {
return {
sanitize: () => {
called_sanitizers.push('_categories');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_geonames_warnings': () => {
'../sanitizer/_geonames_warnings': function () {
return {
sanitize: () => {
called_sanitizers.push('_geonames_warnings');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_location_bias': (defaultParameters) => {
'../sanitizer/_location_bias': function (defaultParameters) {
return {
sanitize: () => {
if (defaultParameters.key === 'value'){
return () => {
called_sanitizers.push('_location_bias');
return { errors: [], warnings: [] };
};
} else {
throw new Error('incorrect parameter passed to _location_bias');
}
}
};
}
});
@ -113,6 +149,7 @@ module.exports.tests.sanitize = (test, common) => {
const res = {};
const middleware = search.middleware({
// mock pelias config ap.defaultParameters section for location bias
defaultParameters: {
key: 'value'
}
@ -124,6 +161,7 @@ module.exports.tests.sanitize = (test, common) => {
});
});
};
module.exports.all = (tape, common) => {

197
test/unit/sanitizer/search_fallback.js
Unescape View File

@ -0,0 +1,197 @@
var proxyquire = require('proxyquire').noCallThru();
module.exports.tests = {};
module.exports.tests.sanitize = function(test, common) {
test('verify that all sanitizers were called as expected when `res` is undefined', function(t) {
var called_sanitizers = [];
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/search_fallback', {
'../sanitizer/_text_addressit': function () {
return {
sanitize: () => {
called_sanitizers.push('_text_addressit');
return { errors: [], warnings: [] };
}
};
}
});
var expected_sanitizers = [
'_text_addressit'
];
var req = {};
search.middleware(req, undefined, function(){
t.deepEquals(called_sanitizers, expected_sanitizers);
t.end();
});
});
test('verify that all sanitizers were called as expected when `res` has no `data` property', function(t) {
var called_sanitizers = [];
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/search_fallback', {
'../sanitizer/_text_addressit': function () {
return {
sanitize: () => {
called_sanitizers.push('_text_addressit');
return { errors: [], warnings: [] };
}
};
},
});
var expected_sanitizers = [
'_text_addressit'
];
var req = {};
var res = {};
search.middleware(req, res, function(){
t.deepEquals(called_sanitizers, expected_sanitizers);
t.end();
});
});
test('verify that all sanitizers were called as expected when res.data is empty', function(t) {
var called_sanitizers = [];
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/search_fallback', {
'../sanitizer/_text_addressit': function () {
return {
sanitize: () => {
called_sanitizers.push('_text_addressit');
return { errors: [], warnings: [] };
}
};
},
});
var expected_sanitizers = [
'_text_addressit'
];
var req = {};
var res = {
data: []
};
search.middleware(req, res, function(){
t.deepEquals(called_sanitizers, expected_sanitizers);
t.end();
});
});
test('non-empty res.data should not call the _text_autocomplete sanitizer', function(t) {
var called_sanitizers = [];
// rather than re-verify the functionality of all the sanitizers, this test just verifies that they
// were all called correctly
var search = proxyquire('../../../sanitizer/search_fallback', {
'../sanitizer/_text_autocomplete': function() {
throw new Error('_text_autocomplete sanitizer should not have been called');
}
});
var expected_sanitizers = [];
var req = {};
var res = {
data: [{}]
};
search.middleware(req, res, function(){
t.deepEquals(called_sanitizers, expected_sanitizers);
t.end();
});
});
test('req.clean.text should be logged when isDNT=false', (t) => {
const infoLog = [];
const search = proxyquire('../../../sanitizer/search_fallback', {
'pelias-logger': {
get: () => {
return {
info: (msg) => {
infoLog.push(msg);
}
};
}
},
'../helper/logging': {
isDNT: () => { return false; }
}
});
const req = {
path: '/v1/search',
clean: {
text: 'this is the query text'
}
};
search.middleware(req, undefined, () => {
t.deepEquals(infoLog, [`fallback queryText: ${req.clean.text}`]);
t.end();
});
});
test('req.clean.text should not be logged when isDNT=true', (t) => {
const infoLog = [];
const search = proxyquire('../../../sanitizer/search_fallback', {
'pelias-logger': {
get: () => {
return {
info: (msg) => {
infoLog.push(msg);
}
};
}
},
'../helper/logging': {
isDNT: () => { return true; }
}
});
const req = {
path: '/v1/search',
clean: {
text: 'this is the query text'
}
};
search.middleware(req, undefined, () => {
t.deepEquals(infoLog, ['fallback queryText: [text removed]']);
t.end();
});
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANITIZE /search_fallback ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

71
test/unit/sanitizer/structured_geocoding.js
Unescape View File

@ -1,4 +1,5 @@
var proxyquire = require('proxyquire').noCallThru();
const _ = require('lodash');
module.exports.tests = {};
@ -10,89 +11,125 @@ module.exports.tests.sanitize = function(test, common) {
// were all called correctly
var search = proxyquire('../../../sanitizer/structured_geocoding', {
'../sanitizer/_deprecate_quattroshapes': function () {
return {
sanitize: () => {
called_sanitizers.push('_deprecate_quattroshapes');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_single_scalar_parameters': function () {
return {
sanitize: () => {
called_sanitizers.push('_single_scalar_parameters');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_synthesize_analysis': function () {
return {
sanitize: () => {
called_sanitizers.push('_synthesize_analysis');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_iso2_to_iso3': function () {
return {
sanitize: () => {
called_sanitizers.push('_iso2_to_iso3');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_city_name_standardizer': function () {
return {
sanitize: () => {
called_sanitizers.push('_city_name_standardizer');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_size': function () {
if (arguments.length === 0) {
return function() {
if (_.isEmpty(arguments)) {
return {
sanitize: () => {
called_sanitizers.push('_size');
return { errors: [], warnings: [] };
}
};
} else {
throw new Error('should not have passed any parameters to _size');
}
},
'../sanitizer/_targets': function (type) {
if (['layers', 'sources'].indexOf(type) !== -1) {
return function() {
called_sanitizers.push('_targets/' + type);
return {
sanitize: () => {
called_sanitizers.push(`_targets/${type}`);
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _targets');
}
},
'../sanitizer/_sources_and_layers': function () {
return {
sanitize: () => {
called_sanitizers.push('_sources_and_layers');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_flag_bool': function () {
if (arguments[0] === 'private' && arguments[1] === false) {
return function() {
return {
sanitize: () => {
called_sanitizers.push('_flag_bool');
return { errors: [], warnings: [] };
};
}
else {
};
} else {
throw new Error('incorrect parameters passed to _flag_bool');
}
},
'../sanitizer/_geo_search': function () {
return {
sanitize: () => {
called_sanitizers.push('_geo_search');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_boundary_country': function () {
return {
sanitize: () => {
called_sanitizers.push('_boundary_country');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_categories': function () {
return {
sanitize: () => {
called_sanitizers.push('_categories');
return { errors: [], warnings: [] };
}
};
},
'../sanitizer/_location_bias': function (defaultParameters) {
return {
sanitize: () => {
if (defaultParameters.key === 'value'){
return () => {
called_sanitizers.push('_location_bias');
return { errors: [], warnings: [] };
};
} else {
throw new Error('incorrect parameter passed to _location_bias');
}
}
};
}
});
var expected_sanitizers = [

Write Preview
Loading…
Cancel
Save