Merge pull request #250 from pelias/sources

Sources

helper/layers.js

@@ -1,25 +0,0 @@
-module.exports = function(alias_layers) {
-  // make a copy of the array so, you are not modifying original ref
-  var layers = alias_layers.slice(0);
-
-  // expand aliases
-  var expand_aliases = function(alias, layers, layer_indeces) {
-    var alias_index  = layers.indexOf(alias);
-    if (alias_index !== -1 ) {
-      layers.splice(alias_index, 1);
-      layers = layers.concat(layer_indeces);
-    }
-    return layers;
-  };
-
-  layers = expand_aliases('poi',   layers, ['geoname','osmnode','osmway']);
-  layers = expand_aliases('admin', layers, ['admin0','admin1','admin2','neighborhood','locality','local_admin']);
-  layers = expand_aliases('address', layers, ['osmaddress','openaddresses']);
-
-  // de-dupe
-  layers = layers.filter(function(item, pos) {
-    return layers.indexOf(item) === pos;
-  });
-
-  return layers;
-};

helper/query_parser.js

@@ -1,26 +1,20 @@
 
 var parser     = require('addressit');
 var extend     = require('extend');
-var get_layers_helper = require('../helper/layers');
+var layers_map = require('../query/layers');
 var delim      = ',';
 
 module.exports = {};
 
 module.exports.get_layers = function get_layers(query) {
-  var tokenized = query.split(/[ ,]+/);
-  var hasNumber = /\d/.test(query);
-
   if (query.length <= 3 ) {
     // no address parsing required
-    return get_layers_helper(['admin']);
+    return layers_map.coarse;
   }
 };
 
 module.exports.get_parsed_address = function get_parsed_address(query) {
 
-  var tokenized = query.split(/[ ,]+/);
-  var hasNumber = /\d/.test(query);
-
   var getAdminPartsBySplittingOnDelim = function(query) {
     // naive approach - for admin matching during query time
     // split 'flatiron, new york, ny' into 'flatiron' and 'new york, ny'

helper/types.js

@@ -9,23 +9,29 @@ var intersection = function intersection(set1, set2) {
   });
 };
 
+/**
+ * Combine all types and determine the unique subset
+ *
+ * @param {Array} clean_types
+ * @returns {Array}
+ */
 module.exports = function calculate_types(clean_types) {
-  if (!clean_types) {
+  if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_address_parser)) {
-    return undefined;
+    throw new Error('clean_types should not be null or undefined');
   }
 
   /* the layers and source parameters are cumulative:
-   * perform a set insersection of their specified types
+   * perform a set intersection of their specified types
    */
-  if (clean_types.from_layers || clean_types.from_source) {
+  if (clean_types.from_layers || clean_types.from_sources) {
     var types = valid_types;
 
     if (clean_types.from_layers) {
       types = intersection(types, clean_types.from_layers);
     }
 
-    if (clean_types.from_source) {
+    if (clean_types.from_sources) {
-      types = intersection(types, clean_types.from_source);
+      types = intersection(types, clean_types.from_sources);
     }
 
     return types;
@@ -38,4 +44,6 @@ module.exports = function calculate_types(clean_types) {
   if (clean_types.from_address_parser) {
     return clean_types.from_address_parser;
   }
+
+  throw new Error('no types specified');
 };

middleware/_types.js

@@ -8,17 +8,27 @@ var types_helper = require( '../helper/types' );
  * message instead of searching at all.
  */
 function middleware(req, res, next) {
+  req.clean = req.clean || {};
+
+  if (req.clean.hasOwnProperty('types') === false) {
+    return next();
+  }
+
+  try {
     var types = types_helper(req.clean.types);
 
-  if (types !== undefined && types.length !== undefined) {
+    if ((types instanceof Array) && types.length === 0) {
-   if (types.length === 0) {
+      var err = 'You have specified both the `sources` and `layers` ' +
-    var err = 'You have specified both the `source` and `layers` ' +
         'parameters in a combination that will return no results.';
       res.status(400); // 400 Bad Request
       return next(err);
-   } else {
+    }
+
     req.clean.type = types;
   }
+  catch (err) {
+    // this means there were no types specified
+    delete req.clean.types;
   }
 
   next();

package.json

@@ -44,6 +44,7 @@
     "geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "geopipes-elasticsearch-backend": "^0.2.0",
+    "lodash": "^3.10.1",
     "markdown": "0.5.0",
     "microtime": "1.4.0",
     "morgan": "1.5.2",

query/layers.js

@@ -0,0 +1,15 @@
+/*
+ * Mapping from data layers to type values
+ */
+
+module.exports = {
+  'venue': ['geoname','osmnode','osmway'],
+  'address': ['osmaddress','openaddresses'],
+  'country': ['admin0'],
+  'region': ['admin1'],
+  'county': ['admin2'],
+  'locality': ['locality'],
+  'localadmin': ['local_admin'],
+  'neighbourhood': ['neighborhood'],
+  'coarse': ['admin0','admin1','admin2','neighborhood','locality','local_admin'],
+};

query/sources.js

@@ -3,8 +3,12 @@
  */
 
 module.exports = {
-  'geonames': ['geoname'],
+  'gn'            : ['geoname'],
+  'geonames'      : ['geoname'],
+  'oa'            : ['openaddresses'],
   'openaddresses' : ['openaddresses'],
-  'quattroshapes': ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'qs'            : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'quattroshapes' : ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'osm'           : ['osmaddress', 'osmnode', 'osmway'],
   'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
 };

routes/v1.js

@@ -70,6 +70,7 @@ function addRoutes(app, peliasConfig) {
     ]),
     reverse: createRouter([
       sanitisers.reverse.middleware,
+      middleware.types,
       controllers.search(undefined, reverseQuery),
       // TODO: add confidence scores
       postProc.distances(),

sanitiser/_targets.js

@@ -0,0 +1,81 @@
+
+var _ = require('lodash'),
+    check = require('check-types');
+
+function setup( paramName, targetMap ) {
+  return function( raw, clean ){
+    return sanitize( raw, clean, {
+      paramName: paramName,
+      targetMap: targetMap,
+      targetMapKeysString: Object.keys(targetMap).join(',')
+    });
+  };
+}
+
+function sanitize( raw, clean, opts ) {
+
+  // error & warning messages
+  var messages = { errors: [], warnings: [] };
+
+  // init clean.types
+  clean.types = clean.types || {};
+
+  // the string of targets (comma delimeted)
+  var targetsString = raw[opts.paramName];
+
+  // trim whitespace
+  if( check.unemptyString( targetsString ) ){
+    targetsString = targetsString.trim();
+
+    // param must be a valid non-empty string
+    if( !check.unemptyString( targetsString ) ){
+      messages.errors.push(
+        opts.paramName + ' parameter cannot be an empty string. Valid options: ' + opts.targetMapKeysString
+      );
+    }
+    else {
+
+      // split string in to array and lowercase each target string
+      var targets = targetsString.split(',').map( function( target ){
+        return target.toLowerCase(); // lowercase inputs
+      });
+
+      // emit an error for each target *not* present in the targetMap
+      targets.filter( function( target ){
+        return !opts.targetMap.hasOwnProperty(target);
+      }).forEach( function( target ){
+        messages.errors.push(
+          '\'' + target + '\' is an invalid ' + opts.paramName + ' parameter. Valid options: ' + opts.targetMapKeysString
+        );
+      });
+
+      // only set types value when no error occured
+      if( !messages.errors.length ){
+
+        // store the values under a new key as 'clean.types.from_*'
+        var typesKey = 'from_' + opts.paramName;
+
+        // ?
+        clean.types[typesKey] = targets.reduce(function(acc, target) {
+          return acc.concat(opts.targetMap[target]);
+        }, []);
+
+        // dedupe in case aliases expanded to common things or user typed in duplicates
+        clean.types[typesKey] = _.unique(clean.types[typesKey]);
+      }
+
+    }
+  }
+
+  // string is empty
+  else if( check.string( targetsString ) ){
+    messages.errors.push(
+      opts.paramName + ' parameter cannot be an empty string. Valid options: ' + opts.targetMapKeysString
+    );
+  }
+
+
+  return messages;
+}
+
+module.exports = setup;

sanitiser/place.js

@@ -9,6 +9,7 @@ var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
 // export sanitize for testing
 module.exports.sanitize = sanitize;
+module.exports.sanitiser_list = sanitizers;
 
 // middleware
 module.exports.middleware = function( req, res, next ){
@@ -17,7 +18,6 @@ module.exports.middleware = function( req, res, next ){
       res.status(400); // 400 Bad Request
       return next(err);
     }
-    req.clean = clean;
     next();
   });
 };

sanitiser/reverse.js

@@ -1,9 +1,9 @@
 
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
-      layers: require('../sanitiser/_layers'),
+      layers: require('../sanitiser/_targets')('layers', require('../query/layers')),
+      sources: require('../sanitiser/_targets')('sources', require('../query/sources')),
       size: require('../sanitiser/_size'),
-      source: require('../sanitiser/_source'),
       details: require('../sanitiser/_details'),
       geo_reverse: require('../sanitiser/_geo_reverse'),
       categories: require('../sanitiser/_categories')
@@ -13,6 +13,7 @@ var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
 // export sanitize for testing
 module.exports.sanitize = sanitize;
+module.exports.sanitiser_list = sanitizers;
 
 // middleware
 module.exports.middleware = function( req, res, next ){
@@ -21,7 +22,7 @@ module.exports.middleware = function( req, res, next ){
       res.status(400); // 400 Bad Request
       return next(err);
     }
-    req.clean = clean;
     next();
   });
 };
+

sanitiser/search.js

@@ -2,17 +2,19 @@
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       text: require('../sanitiser/_text'),
-      layers: require('../sanitiser/_layers'),
       size: require('../sanitiser/_size'),
-      source: require('../sanitiser/_source'),
+      layers: require('../sanitiser/_targets')('layers', require( '../query/layers' )),
+      sources: require('../sanitiser/_targets')('sources', require( '../query/sources' )),
       details: require('../sanitiser/_details'),
-      geo_search: require('../sanitiser/_geo_search')
+      geo_search: require('../sanitiser/_geo_search'),
+      categories: require('../sanitiser/_categories')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
 // export sanitize for testing
 module.exports.sanitize = sanitize;
+module.exports.sanitiser_list = sanitizers;
 
 // middleware
 module.exports.middleware = function( req, res, next ){
@@ -21,7 +23,7 @@ module.exports.middleware = function( req, res, next ){
       res.status(400); // 400 Bad Request
       return next(err);
     }
-    req.clean = clean;
     next();
   });
 };
+

test/unit/helper/query_parser.js

@@ -1,6 +1,6 @@
 
 var parser = require('../../../helper/query_parser');
-var get_layers = require('../../../helper/layers');
+var layers_map = require('../../../query/layers');
 
 module.exports.tests = {};
 
@@ -43,7 +43,7 @@ module.exports.tests.parse_three_chars_or_less = function(test, common) {
   var testParse = function(query) {
     test('query length < 3 (' + query + ')', function(t) {
       var address = parser.get_parsed_address(query);
-      var target_layer = get_layers(['admin']);
+      var target_layer = layers_map.coarse;
       var layers = parser.get_layers(query);
 
       t.equal(typeof address, 'object', 'valid object');
@@ -67,7 +67,7 @@ module.exports.tests.parse_one_or_more_tokens = function(test, common) {
   var testParse = function(query, parse_address) {
     test('query with one or more tokens (' + query + ')', function(t) {
       var address = parser.get_parsed_address(query);
-      var target_layer = get_layers(['admin', 'poi']);
+      var target_layer = layers_map.coarse.concat(layers_map.venue);
       var layers = parser.get_layers(query);
 
       t.equal(typeof address, 'object', 'valid object');
@@ -119,7 +119,6 @@ module.exports.tests.parse_address = function(test, common) {
       query_string = query_string.substring(1);
 
       var address = parser.get_parsed_address(query_string);
-      var non_address_layer = get_layers(['admin', 'poi']);
 
       t.equal(typeof address, 'object', 'valid object for the address ('+query_string+')');
 

test/unit/helper/types.js

@@ -1,20 +1,32 @@
 var types = require('../../../helper/types');
 
-var valid_types = require( '../../../query/types' );
 module.exports.tests = {};
 
 module.exports.tests.no_cleaned_types = function(test, common) {
   test('no cleaned types', function(t) {
-    var actual = types(undefined);
+    try {
-    t.equal(actual, undefined, 'all valid types returned for empty input');
+      types();
+      t.fail('exception should be thrown');
+    }
+    catch (err) {
+      t.equal(err.message, 'clean_types should not be null or undefined', 'no input should result in exception');
+    }
+    finally {
       t.end();
+    }
   });
 
   test('no cleaned types', function(t) {
-    var cleaned_types = {};
+    try {
-    var actual = types(cleaned_types);
+      types({});
-    t.equal(actual, undefined, 'all valid types returned for empty input');
+      t.fail('exception should be thrown');
+    }
+    catch (err) {
+      t.equal(err.message, 'clean_types should not be null or undefined', 'no input should result in exception');
+    }
+    finally {
       t.end();
+    }
   });
 };
 
@@ -58,7 +70,7 @@ module.exports.tests.layers_parameter_and_address_parser = function(test, common
 module.exports.tests.source_parameter = function(test, common) {
   test('source parameter specified', function(t) {
     var cleaned_types = {
-      from_source: ['openaddresses']
+      from_sources: ['openaddresses']
     };
 
     var actual = types(cleaned_types);
@@ -72,7 +84,7 @@ module.exports.tests.source_parameter = function(test, common) {
 module.exports.tests.source_and_layers_parameters = function(test, common) {
   test('source and layers parameter both specified', function(t) {
     var cleaned_types = {
-      from_source: ['openaddresses'],
+      from_sources: ['openaddresses'],
       from_layers: ['osmaddress', 'openaddresses']
     };
 

test/unit/run.js

@@ -8,8 +8,9 @@ var tests = [
   require('./controller/search'),
   require('./service/mget'),
   require('./service/search'),
-  require('./sanitiser/_source'),
+  require('./sanitiser/_sources'),
   require('./sanitiser/search'),
+  require('./sanitiser/_layers'),
   require('./sanitiser/reverse'),
   require('./sanitiser/place'),
   require('./query/types'),

test/unit/sanitiser/_layers.js

@@ -0,0 +1,120 @@
+
+var sanitize = require('../../../sanitiser/_targets')('layers', require('../../../query/layers'));
+
+module.exports.tests = {};
+
+module.exports.tests.sanitize_layers = function(test, common) {
+  test('unspecified', function(t) {
+    var messages = sanitize({ layers: undefined }, {});
+    t.equal(messages.errors.length, 0, 'no errors');
+    t.end();
+  });
+  test('invalid layer', function(t) {
+    var raw = { layers: 'test_layer' };
+    var clean = {};
+
+    var messages = sanitize(raw, clean);
+
+    var msg = ' is an invalid layers parameter. Valid options: ';
+    t.equal(messages.errors.length, 1, 'errors set');
+    t.true(messages.errors[0].match(msg), 'invalid layer requested');
+    t.true(messages.errors[0].length > msg.length, 'invalid error message');
+    t.end();
+  });
+  test('venue (alias) layer', function(t) {
+    var venue_layers = ['geoname','osmnode','osmway'];
+    var raw = { layers: 'venue' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers set');
+    t.end();
+  });
+  test('coarse (alias) layer', function(t) {
+    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
+    var raw = { layers: 'coarse' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, admin_layers, 'coarse layers set');
+    t.end();
+  });
+  test('address (alias) layer', function(t) {
+    var address_layers = ['osmaddress','openaddresses'];
+    var raw = { layers: 'address' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, address_layers, 'address layers set');
+    t.end();
+  });
+  test('venue alias layer plus regular layers', function(t) {
+    var venue_layers = ['geoname','osmnode','osmway'];
+    var reg_layers = ['admin0', 'admin1'];
+    var raw = { layers: 'venue,country,region' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, venue_layers.concat(reg_layers), 'venue + regular layers');
+    t.end();
+  });
+  test('coarse alias layer plus regular layers', function(t) {
+    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
+    var reg_layers   = ['geoname', 'osmnode', 'osmway'];
+
+    var raw = { layers: 'coarse,venue,country' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, admin_layers.concat(reg_layers), 'coarse + regular layers set');
+    t.end();
+  });
+  test('address alias layer plus regular layers', function(t) {
+    var address_layers = ['osmaddress','openaddresses'];
+    var reg_layers   = ['admin0', 'locality'];
+
+    var raw = { layers: 'address,country,locality' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, address_layers.concat(reg_layers), 'address + regular layers set');
+    t.end();
+  });
+  test('alias layer plus regular layers (no duplicates)', function(t) {
+    var venue_layers = ['geoname','osmnode','osmway','admin0'];
+    var raw = { layers: 'venue,country' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, venue_layers, 'venue layers found (no duplicates)');
+    t.end();
+  });
+  test('multiple alias layers (no duplicates)', function(t) {
+    var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
+    var raw = { layers: 'venue,coarse' };
+    var clean = {};
+
+    sanitize(raw, clean);
+
+    t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SANTIZE _layers ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/_sources.js

@@ -0,0 +1,132 @@
+var sanitize = require( '../../../sanitiser/_targets' )('sources', require('../../../query/sources'));
+
+var success_messages = { error: false };
+
+module.exports.tests = {};
+
+module.exports.tests.no_sources = function(test, common) {
+  test('sources is not set', function(t) {
+    var req = {
+      query: { },
+      clean: { }
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+
+  test('source is empty string', function(t) {
+    var req = {
+      query: {
+        sources: ''
+      },
+      clean: { }
+    };
+
+    var expected_error = 'sources parameter cannot be an empty string. ' +
+       'Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap';
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
+    t.deepEqual(messages.errors.length, 1, 'error returned');
+    t.deepEqual(messages.errors[0], expected_error, 'error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+};
+
+module.exports.tests.valid_sources = function(test, common) {
+  test('geonames source', function(t) {
+    var req = {
+      query: {
+        sources: 'geonames'
+      },
+      clean: { }
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(req.clean.types, { from_sources: ['geoname'] }, 'clean.types should contain from_source entry with geonames');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+
+    t.end();
+  });
+
+  test('openstreetmap source', function(t) {
+    var req = {
+      query: {
+        sources: 'openstreetmap'
+      },
+      clean: { }
+    };
+    var expected_types = {
+      from_sources: ['osmaddress', 'osmnode', 'osmway']
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(req.clean.types, expected_types, 'clean.types should contain from_source entry with multiple entries for openstreetmap');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+
+  test('multiple sources', function(t) {
+    var req = {
+      query: {
+        sources: 'openstreetmap,openaddresses'
+      },
+      clean: { }
+    };
+    var expected_types = {
+      from_sources: ['osmaddress', 'osmnode', 'osmway', 'openaddresses']
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(req.clean.types, expected_types,
+                'clean.types should contain from_source entry with multiple entries for openstreetmap and openadresses');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+};
+
+module.exports.tests.invalid_sources = function(test, common) {
+  test('geonames source', function(t) {
+    var req = {
+      query: {
+        sources: 'notasource'
+      },
+      clean: { }
+    };
+    var expected_messages = {
+      errors: [
+        '\'notasource\' is an invalid sources parameter. Valid options: gn,geonames,oa,openaddresses,qs,quattroshapes,osm,openstreetmap'
+      ],
+      warnings: []
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.deepEqual(messages, expected_messages, 'error with message returned');
+    t.deepEqual(req.clean.types, { }, 'clean.types should remain empty');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANTIZE _sources ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/reverse.js

@@ -1,8 +1,7 @@
 
-var suggest  = require('../../../sanitiser/reverse'),
+var reverse  = require('../../../sanitiser/reverse'),
-    _sanitize = suggest.sanitize,
+    _sanitize = reverse.sanitize,
-    middleware = suggest.middleware,
+    middleware = reverse.middleware,
-    delim = ',',
     defaultError = 'missing param \'lat\'',
     defaultClean =  { lat:0,
                       types: {
@@ -29,6 +28,14 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
+module.exports.tests.sanitisers = function(test, common) {
+  test('check sanitiser list', function (t) {
+    var expected = ['layers', 'sources', 'size', 'details', 'geo_reverse', 'categories'];
+    t.deepEqual(Object.keys(reverse.sanitiser_list), expected);
+    t.end();
+  });
+};
+
 module.exports.tests.sanitize_lat = function(test, common) {
   var lats = {
     invalid: [],
@@ -154,82 +161,6 @@ module.exports.tests.sanitize_details = function(test, common) {
   });
 };
 
-module.exports.tests.sanitize_layers = function(test, common) {
-  test('unspecified', function(t) {
-    sanitize({ layers: undefined, 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
-      t.end();
-    });
-  });
-  test('invalid layer', function(t) {
-    sanitize({ layers: 'test_layer', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      var msg = 'invalid param \'layers\': must be one or more of ';
-      t.true(err.match(msg), 'invalid layer requested');
-      t.true(err.length > msg.length, 'invalid error message');
-      t.end();
-    });
-  });
-  test('poi (alias) layer', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    sanitize({ layers: 'poi', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
-      t.end();
-    });
-  });
-  test('admin (alias) layer', function(t) {
-    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    sanitize({ layers: 'admin', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
-      t.end();
-    });
-  });
-  test('address (alias) layer', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
-    sanitize({ layers: 'address', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, address_layers, 'address layers set');
-      t.end();
-    });
-  });
-  test('poi alias layer plus regular layers', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    var reg_layers = ['admin0', 'admin1'];
-    sanitize({ layers: 'poi,admin0,admin1', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
-      t.end();
-    });
-  });
-  test('admin alias layer plus regular layers', function(t) {
-    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    var reg_layers   = ['geoname', 'osmway'];
-    sanitize({ layers: 'admin,geoname,osmway', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
-      t.end();
-    });
-  });
-  test('address alias layer plus regular layers', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
-    var reg_layers   = ['geoname', 'osmway'];
-    sanitize({ layers: 'address,geoname,osmway', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
-      t.end();
-    });
-  });
-  test('alias layer plus regular layers (no duplicates)', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    sanitize({ layers: 'poi,geoname,osmnode', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
-      t.end();
-    });
-  });
-  test('multiple alias layers (no duplicates)', function(t) {
-    var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    sanitize({ layers: 'poi,admin', 'point.lat': 0, 'point.lon': 0 }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
-      t.end();
-    });
-  });
-};
-
 module.exports.tests.sanitize_categories = function(test, common) {
   var queryParams = { 'point.lat': 0, 'point.lon': 0 };
   test('unspecified', function(t) {

test/unit/sanitiser/search.js

@@ -5,7 +5,6 @@ var search  = require('../../../sanitiser/search'),
     defaultParsed = _text.defaultParsed,
     _sanitize = search.sanitize,
     middleware = search.middleware,
-    delim = ',',
     defaultError = 'invalid param \'text\': text length, must be >0',
     defaultClean =  { text: 'test',
                       types: {
@@ -26,7 +25,15 @@ module.exports.tests.interface = function(test, common) {
   });
   test('middleware interface', function(t) {
     t.equal(typeof middleware, 'function', 'middleware is a function');
-    t.equal(middleware.length, 3, 'sanitizee has a valid middleware');
+    t.equal(middleware.length, 3, 'sanitize has a valid middleware');
+    t.end();
+  });
+};
+
+module.exports.tests.sanitisers = function(test, common) {
+  test('check sanitiser list', function (t) {
+    var expected = ['text', 'size', 'layers', 'sources', 'details', 'geo_search', 'categories' ];
+    t.deepEqual(Object.keys(search.sanitiser_list), expected);
     t.end();
   });
 };
@@ -273,91 +280,6 @@ module.exports.tests.sanitize_details = function(test, common) {
   });
 };
 
-module.exports.tests.sanitize_layers = function(test, common) {
-  test('unspecified', function(t) {
-    sanitize({ layers: undefined, text: 'test' }, function( err, clean ){
-      t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
-      t.end();
-    });
-  });
-  test('invalid layer', function(t) {
-    sanitize({ layers: 'test_layer', text: 'test' }, function( err, clean ){
-      var msg = 'invalid param \'layers\': must be one or more of ';
-      t.true(err.match(msg), 'invalid layer requested');
-      t.true(err.length > msg.length, 'invalid error message');
-      t.end();
-    });
-  });
-  test('poi (alias) layer', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    sanitize({ layers: 'poi', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
-      t.end();
-    });
-  });
-  test('admin (alias) layer', function(t) {
-    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    sanitize({ layers: 'admin', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
-      t.end();
-    });
-  });
-  test('address (alias) layer', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
-    sanitize({ layers: 'address', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, address_layers, 'types from layers set');
-      t.deepEqual(clean.types.from_address_parser, _text.allLayers, 'address parser uses default layers');
-      t.end();
-    });
-  });
-  test('poi alias layer plus regular layers', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    var reg_layers = ['admin0', 'admin1'];
-    sanitize({ layers: 'poi,admin0,admin1', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
-      t.end();
-    });
-  });
-  test('admin alias layer plus regular layers', function(t) {
-    var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    var reg_layers   = ['geoname', 'osmway'];
-    sanitize({ layers: 'admin,geoname,osmway', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
-      t.end();
-    });
-  });
-  test('address alias layer plus regular layers', function(t) {
-    var address_layers = ['osmaddress','openaddresses'];
-    var reg_layers   = ['geoname', 'osmway'];
-    sanitize({ layers: 'address,geoname,osmway', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
-      t.end();
-    });
-  });
-  test('alias layer plus regular layers (no duplicates)', function(t) {
-    var poi_layers = ['geoname','osmnode','osmway'];
-    sanitize({ layers: 'poi,geoname,osmnode', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
-      t.end();
-    });
-  });
-  test('multiple alias layers (no duplicates)', function(t) {
-    var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
-    sanitize({ layers: 'poi,admin', text: 'test' }, function( err, clean ){
-      t.equal(err, undefined);
-      t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
-      t.end();
-    });
-  });
-};
-
 module.exports.tests.invalid_params = function(test, common) {
   test('invalid text params', function(t) {
     sanitize( undefined, function( err, clean ){