Merge branch 'source_parameter'

controller/search.js

@@ -1,4 +1,3 @@
-
 var service = { search: require('../service/search') };
 
 function setup( backend, query ){
@@ -16,13 +15,8 @@ function setup( backend, query ){
       body: query( req.clean )
     };
 
-    if (req.clean.layers) {
-      cmd.type = req.clean.layers;
-    }
-
-    // set type if input suggests targeting a layer(s)
-    if (req.clean.default_layers_set && req.clean.parsed_input) {
-      cmd.type = req.clean.parsed_input.target_layer || cmd.type;
+    if ( req.clean.hasOwnProperty('type') ) {
+      cmd.type = req.clean.type;
     }
 
     // query backend

helper/query_parser.js

@@ -1,10 +1,25 @@
 
 var parser     = require('addressit');
 var extend     = require('extend');
-var get_layers = require('../helper/layers');
+var get_layers_helper = require('../helper/layers');
 var delim      = ',';
 
-module.exports = function(query) {
+module.exports = {};
+
+module.exports.get_layers = function get_layers(query) {
+  var tokenized = query.split(/[ ,]+/);
+  var hasNumber = /\d/.test(query);
+
+  if (query.length <= 3 ) {
+    // no address parsing required
+    return get_layers_helper(['admin']);
+  } else if (tokenized.length === 1 || (tokenized.length < 3 && !hasNumber)) {
+    // no need to hit address layers if there's only one (or two) token(s)
+    return get_layers_helper(['admin', 'poi']);
+  }
+};
+
+module.exports.get_parsed_address = function get_parsed_address(query) {
 
   var tokenized = query.split(/[ ,]+/);
   var hasNumber = /\d/.test(query);
@@ -22,37 +37,18 @@ module.exports = function(query) {
     return address;
   };
 
-  var getTargetLayersWhenAddressParsingIsNotNecessary = function(query) {
-    var address = {};
-    // set target_layer if input length <= 3 characters
-    if (query.length <= 3 ) {
-      // no address parsing required
-      address.target_layer = get_layers(['admin']);
-    } else if (tokenized.length === 1 || (tokenized.length < 3 && !hasNumber)) {
-      // no need to hit address layers if there's only one (or two) token(s)
-      address.target_layer = get_layers(['admin', 'poi']);
-    }
-    
-    return address.target_layer ? address : null;
-  };
-
   var getAddressParts = function(query) {
-    // address parsing
-    var address = parser( query );
-    // set target_layer if input suggests no address
-    if (address.text === address.regions.join(' ') && !hasNumber) {
-      address.target_layer = get_layers(['admin', 'poi']);
+    // perform full address parsing
+    // except on queries so short they obviously can't contain an address
+    if (query.length > 3) {
+      return parser( query );
     }
-
-    return address;
   };
 
   var addressWithAdminParts  = getAdminPartsBySplittingOnDelim(query);
-  var addressWithTargetLayers= getTargetLayersWhenAddressParsingIsNotNecessary(query);
-  var addressWithAddressParts= !addressWithTargetLayers ? getAddressParts(query) : {}; 
+  var addressWithAddressParts= getAddressParts(query);
 
   var parsedAddress  = extend(addressWithAdminParts,
-                              addressWithTargetLayers, 
                               addressWithAddressParts);
 
   var address_parts  =  [ 'name',
@@ -63,8 +59,7 @@ module.exports = function(query) {
                           'country',
                           'postalcode',
                           'regions',
-                          'admin_parts',
-                          'target_layer'
+                          'admin_parts'
                         ];
 
   var parsed_input = {};
@@ -77,17 +72,3 @@ module.exports = function(query) {
 
   return parsed_input;
 };
-
-
-// parsed_input = {
-//   name   : parsedAddress.name,
-//   number : parsedAddress.number,
-//   street : parsedAddress.street,
-//   city   : parsedAddress.city,
-//   state  : parsedAddress.state,
-//   country: parsedAddress.country,
-//   postalcode : parsedAddress.postalcode,
-//   regions: parsedAddress.regions,
-//   admin_parts: parsedAddress.admin_parts,
-//   target_layer: parsedAddress.target_layer
-// }

helper/types.js

@@ -0,0 +1,41 @@
+var valid_types = require( '../query/types' );
+
+/**
+ * Calculate the set-style intersection of two arrays
+ */
+var intersection = function intersection(set1, set2) {
+  return set2.filter(function(value) {
+      return set1.indexOf(value) !== -1;
+  });
+};
+
+module.exports = function calculate_types(clean_types) {
+  if (!clean_types) {
+    return undefined;
+  }
+
+  /* the layers and source parameters are cumulative:
+   * perform a set insersection of their specified types
+   */
+  if (clean_types.from_layers || clean_types.from_source) {
+    var types = valid_types;
+
+    if (clean_types.from_layers) {
+      types = intersection(types, clean_types.from_layers);
+    }
+
+    if (clean_types.from_source) {
+      types = intersection(types, clean_types.from_source);
+    }
+
+    return types;
+  }
+
+  /*
+   * Type restrictions requested by the address parser should only be used
+   * if both the source and layers parameters are empty, so do this last
+   */
+  if (clean_types.from_address_parser) {
+    return clean_types.from_address_parser;
+  }
+};

middleware/_types.js

@@ -0,0 +1,27 @@
+var types_helper = require( '../helper/types' );
+
+/**
+ * Validate the types specified to be searched.
+ *
+ * Elasticsearch interprets an empty array of types as "search anything" rather
+ * than "search nothing", so in the case of an empty array, return an error
+ * message instead of searching at all.
+ */
+function middleware(req, res, next) {
+  var types = types_helper(req.clean.types);
+
+  if (types !== undefined && types.length !== undefined) {
+   if (types.length === 0) {
+    var err = 'You have specified both the `source` and `layers` ' +
+    'parameters in a combination that will return no results.';
+    res.status(400); // 400 Bad Request
+    return next(err);
+   } else {
+     req.clean.type = types;
+   }
+  }
+
+  next();
+}
+
+module.exports = middleware;

query/sources.js

@@ -0,0 +1,10 @@
+/*
+ * Mapping from data sources to type values
+ */
+
+module.exports = {
+  'geonames': ['geoname'],
+  'openaddresses' : ['openaddresses'],
+  'quattroshapes': ['admin0', 'admin1', 'admin2', 'neighborhood', 'locality', 'local_admin'],
+  'openstreetmap' : ['osmaddress', 'osmnode', 'osmway']
+};

query/types.js

@@ -1,5 +1,5 @@
 
-// querable indeces
+// querable types
 
 module.exports = [
   'geoname',

routes/v1.js

@@ -9,6 +9,11 @@ var sanitisers = {
   reverse: require('../sanitiser/reverse')
 };
 
+/** ----------------------- middleware ------------------------ **/
+var middleware = {
+  types: require('../middleware/_types')
+};
+
 /** ----------------------- controllers ----------------------- **/
 
 var controllers     = {
@@ -45,6 +50,7 @@ function addRoutes(app, peliasConfig) {
     ]),
     search: createRouter([
       sanitisers.search.middleware,
+      middleware.types,
       controllers.search(),
       postProc.confidenceScores(peliasConfig),
       postProc.renamePlacenames(),

sanitiser/_id.js

@@ -5,10 +5,9 @@ var isObject = require('is-object');
 // so, both type and id are required fields.
 
 function sanitize( req ){
-  
   req.clean   = req.clean || {};
   var params  = req.query;
-  var indeces = require('../query/indeces');
+  var types = require('../query/types');
   var delim   = ':';
 
   // ensure params is a valid object
@@ -56,9 +55,9 @@ function sanitize( req ){
       if('string' !== typeof type || !type.length){
         return errormessage(thisparam);
       }
-      // type text must be one of the indeces
-      if(indeces.indexOf(type) === -1){
-        return errormessage('type', type + ' is invalid. It must be one of these values - [' + indeces.join(', ') + ']');
+      // type text must be one of the types
+      if(types.indexOf(type) === -1){
+        return errormessage('type', type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']');
       }
       req.clean.ids.push({
         id: id,
@@ -68,7 +67,6 @@ function sanitize( req ){
   }
 
   return { 'error': false };
-
 }
 
 // export function

sanitiser/_input.js

@@ -1,9 +1,8 @@
-var isObject   = require('is-object');
-var query_parse= require('../helper/query_parser');
+var isObject     = require('is-object');
+var query_parser = require('../helper/query_parser');
 
 // validate inputs, convert types and apply defaults
 function sanitize( req ){
-  
   req.clean = req.clean || {};
   var params= req.query;
 
@@ -22,11 +21,12 @@ function sanitize( req ){
 
   req.clean.input = params.input;
 
-  req.clean.parsed_input = query_parse(params.input);
+  req.clean.parsed_input = query_parser.get_parsed_address(params.input);
 
+  req.clean.types = req.clean.layers || {};
+  req.clean.types.from_address_parsing = query_parser.get_layers(params.input);
 
   return { 'error': false };
-
 }
 
 // export function

sanitiser/_layers.js

@@ -1,28 +1,29 @@
 
 var isObject = require('is-object'),
-    indeces = require('../query/indeces'),
+    types = require('../query/types'),
     get_layers = require('../helper/layers');
 
 // validate inputs, convert types and apply defaults
 function sanitize( req ){
-  
   var clean = req.clean || {};
   var params= req.query;
 
+  clean.types = clean.types || {};
+
   // ensure the input params are a valid object
   if( !isObject( params ) ){
     params = {};
   }
 
   // default case (no layers specified in GET params)
+  // don't even set the from_layers key in this case
   if('string' !== typeof params.layers || !params.layers.length){
-    params.layers = 'poi,admin,address'; // default layers
-    clean.default_layers_set = true;
+    return { 'error': false };
   }
 
   // decide which layers can be queried
   var alias_layers  = ['poi', 'admin', 'address'];
-  var alias_indeces = indeces.concat(alias_layers);
+  var alias_types = types.concat(alias_layers);
 
   // parse GET params
   var layers = params.layers.split(',').map( function( layer ){
@@ -31,16 +32,16 @@ function sanitize( req ){
 
   // validate layer names
   for( var x=0; x<layers.length; x++ ){
-    if( -1 === alias_indeces.indexOf( layers[x] ) ){
+    if( -1 === alias_types.indexOf( layers[x] ) ){
       return {
         'error': true,
-        'message': 'invalid param \'layers\': must be one or more of ' + alias_indeces.join(',')
+        'message': 'invalid param \'layers\': must be one or more of ' + alias_types.join(',')
       };
     }
   }
 
   // pass validated params to next middleware
-  clean.layers = get_layers(layers);
+  clean.types.from_layers = get_layers(layers);
   req.clean = clean;
 
   return { 'error': false };

sanitiser/_source.js

@@ -0,0 +1,44 @@
+var isObject = require('is-object');
+var sources_map = require( '../query/sources' );
+var all_sources = Object.keys(sources_map);
+
+function sanitize( req ) {
+  req.clean = req.clean || {};
+  var params = req.query;
+
+  req.clean.types = req.clean.types || {};
+
+  // ensure the input params are a valid object
+  if( !isObject( params ) ){
+    params = {};
+  }
+
+  // default case (no layers specified in GET params)
+  // don't even set the from_layers key in this case
+  if('string' !== typeof params.source || !params.source.length){
+    return { error: false };
+  }
+
+  var sources = params.source.split(',');
+
+  var invalid_sources = sources.filter(function(source) {
+    return all_sources.indexOf(source) === -1;
+  });
+
+  if (invalid_sources.length > 0) {
+    return {
+      error: true,
+      msg: '`' + invalid_sources[0] + '` is an invalid source parameter. Valid options: ' + all_sources.join(', ')
+    };
+  }
+
+  var types = sources.reduce(function(acc, source) {
+    return acc.concat(sources_map[source]);
+  }, []);
+
+  req.clean.types.from_source = types;
+
+  return { error: false };
+}
+
+module.exports = sanitize;

sanitiser/search.js

@@ -4,6 +4,7 @@ var _sanitize = require('../sanitiser/_sanitize'),
       input: require('../sanitiser/_input'),
       size: require('../sanitiser/_size'),
       layers: require('../sanitiser/_layers'),
+      source: require('../sanitiser/_source'),
       details: require('../sanitiser/_details'),
       latlonzoom: require('../sanitiser/_geo')
     };

test/unit/helper/query_parser.js

@@ -6,7 +6,8 @@ module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('interface', function(t) {
-    t.equal(typeof parser, 'function', 'valid function');
+    t.equal(typeof parser.get_parsed_address, 'function', 'valid function');
+    t.equal(typeof parser.get_layers, 'function', 'valid function');
     t.end();
   });
 };
@@ -17,7 +18,7 @@ module.exports.tests.split_on_comma = function(test, common) {
 
   var testParse = function(query) {
     test('naive parsing ' + query, function(t) {
-      var address = parser(query);
+      var address = parser.get_parsed_address(query);
       var delimIndex = query.indexOf(delim);
       var name = query.substring(0, delimIndex);
       var admin_parts = query.substring(delimIndex + 1).trim();
@@ -41,11 +42,12 @@ module.exports.tests.parse_three_chars_or_less = function(test, common) {
 
   var testParse = function(query) {
     test('query length < 3 (' + query + ')', function(t) {
-      var address = parser(query);
+      var address = parser.get_parsed_address(query);
       var target_layer = get_layers(['admin']);
+      var layers = parser.get_layers(query);
 
       t.equal(typeof address, 'object', 'valid object');
-      t.deepEqual(address.target_layer, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
+      t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
       t.end();
     });
   };
@@ -63,15 +65,16 @@ module.exports.tests.parse_one_or_more_tokens = function(test, common) {
 
   var testParse = function(query, parse_address) {
     test('query with one or more tokens (' + query + ')', function(t) {
-      var address = parser(query);
+      var address = parser.get_parsed_address(query);
       var target_layer = get_layers(['admin', 'poi']);
+      var layers = parser.get_layers(query);
 
       t.equal(typeof address, 'object', 'valid object');
 
       if (parse_address) {
         t.deepEqual(address.regions.join(''), query, 'since query contained a number, it went through address parsing');
       } else {
-        t.deepEqual(address.target_layer, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));  
+        t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', '));
       }
 
       t.end();
@@ -114,7 +117,7 @@ module.exports.tests.parse_address = function(test, common) {
       // remove leading whitespace
       query_string = query_string.substring(1);
 
-      var address = parser(query_string);
+      var address = parser.get_parsed_address(query_string);
       var non_address_layer = get_layers(['admin', 'poi']);
 
       t.equal(typeof address, 'object', 'valid object for the address ('+query_string+')');
@@ -137,10 +140,6 @@ module.exports.tests.parse_address = function(test, common) {
         t.equal(address.postalcode, query.zip, 'correct postal code (' + query.zip + ')');
       }
 
-      if (address.text === address.regions.join(' ')) {
-        t.deepEqual(address.target_layer, query.target_layer, 'admin_parts set correctly to ' + query.target_layer.join(', '));  
-      }
-      
       t.end();
     });
   };

test/unit/helper/types.js

@@ -0,0 +1,96 @@
+var types = require('../../../helper/types');
+
+var valid_types = require( '../../../query/types' );
+module.exports.tests = {};
+
+module.exports.tests.no_cleaned_types = function(test, common) {
+  test('no cleaned types', function(t) {
+    var actual = types(undefined);
+    t.equal(actual, undefined, 'all valid types returned for empty input');
+    t.end();
+  });
+
+  test('no cleaned types', function(t) {
+    var cleaned_types = {};
+    var actual = types(cleaned_types);
+    t.equal(actual, undefined, 'all valid types returned for empty input');
+    t.end();
+  });
+};
+
+module.exports.tests.address_parser = function(test, common) {
+  test('address parser specifies only admin layers', function(t) {
+    var cleaned_types = {
+      from_address_parser: ['admin0'] // simplified return value from address parser
+    };
+    var actual = types(cleaned_types);
+    var expected = ['admin0']; // simplified expected value for all admin layers
+    t.deepEqual(actual, expected, 'only layers specified by address parser returned');
+    t.end();
+  });
+};
+
+module.exports.tests.layers_parameter = function(test, common) {
+  test('layers parameter specifies only some layers', function(t) {
+    var cleaned_types = {
+      from_layers: ['geoname']
+    };
+    var actual = types(cleaned_types);
+    var expected = ['geoname'];
+    t.deepEqual(actual, expected, 'only types specified by layers parameter returned');
+    t.end();
+  });
+};
+
+module.exports.tests.layers_parameter_and_address_parser = function(test, common) {
+  test('layers parameter and address parser present', function(t) {
+    var cleaned_types = {
+      from_layers: ['geoname'],
+      from_address_parser: ['admin0'] // simplified return value from address parse
+    };
+    var actual = types(cleaned_types);
+    var expected = ['geoname'];
+    t.deepEqual(actual, expected, 'layers parameter overrides address parser completely');
+    t.end();
+  });
+};
+
+module.exports.tests.source_parameter = function(test, common) {
+  test('source parameter specified', function(t) {
+    var cleaned_types = {
+      from_source: ['openaddresses']
+    };
+
+    var actual = types(cleaned_types);
+
+    var expected = ['openaddresses'];
+    t.deepEqual(actual, expected, 'type parameter set to types specified by source');
+    t.end();
+  });
+};
+
+module.exports.tests.source_and_layers_parameters = function(test, common) {
+  test('source and layers parameter both specified', function(t) {
+    var cleaned_types = {
+      from_source: ['openaddresses'],
+      from_layers: ['osmaddress', 'openaddresses']
+    };
+
+    var actual = types(cleaned_types);
+
+    var expected = ['openaddresses'];
+    t.deepEqual(actual, expected, 'type set to intersection of source and layer types');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('types: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/search.js

@@ -274,8 +274,7 @@ module.exports.tests.query = function(test, common) {
                 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], 
       size: 10,
       details: true,
-      parsed_input: parser(address),
-      default_layers_set: true
+      parsed_input: parser.get_parsed_address(address),
     });
 
     var expected = {
@@ -476,8 +475,7 @@ module.exports.tests.query = function(test, common) {
                 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], 
       size: 10,
       details: true,
-      parsed_input: parser(partial_address),
-      default_layers_set: true
+      parsed_input: parser.get_parsed_address(partial_address),
     });
 
     var expected = {
@@ -644,8 +642,7 @@ module.exports.tests.query = function(test, common) {
         'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
       size: 10,
       details: true,
-      parsed_input: parser(partial_address),
-      default_layers_set: true
+      parsed_input: parser.get_parsed_address(partial_address),
     });
 
     var expected = {

test/unit/query/types.js

@@ -1,12 +1,12 @@
 
-var indeces = require('../../../query/indeces');
+var types = require('../../../query/types');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.true(Array.isArray(indeces), 'valid array');
-    t.equal(indeces.length, 11, 'valid array');
+    t.true(Array.isArray(types), 'valid array');
+    t.equal(types.length, 11, 'valid array');
     t.end();
   });
 };
@@ -14,7 +14,7 @@ module.exports.tests.interface = function(test, common) {
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('indeces ' + name, testFunction);
+    return tape('types ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/run.js

@@ -8,10 +8,11 @@ var tests = [
   require('./controller/search'),
   require('./service/mget'),
   require('./service/search'),
+  require('./sanitiser/_source'),
   require('./sanitiser/search'),
   require('./sanitiser/reverse'),
   require('./sanitiser/place'),
-  require('./query/indeces'),
+  require('./query/types'),
   require('./query/sort'),
   require('./query/search'),
   require('./query/reverse'),
@@ -19,6 +20,7 @@ var tests = [
   require('./helper/geojsonify'),
   require('./helper/outputSchema'),
   require('./helper/adminFields'),
+  require('./helper/types'),
 ];
 
 tests.map(function(t) {

test/unit/sanitiser/_input.js

@@ -7,7 +7,7 @@ var input  = require('../../../sanitiser/_input'),
                    'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
     nonAddressLayers = [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 
                    'locality', 'local_admin' ],
-    defaultParsed=  { target_layer: nonAddressLayers },
+    defaultParsed=  { },
     defaultClean =  { input: 'test', 
                       layers: allLayers, 
                       size: 10,
@@ -17,7 +17,7 @@ var input  = require('../../../sanitiser/_input'),
                       lon:0
                     },
     getTargetLayers = function(query) {
-      var address = parser(query);
+      var address = parser.get_parsed_address(query);
       return address.target_layer;
     };
 

test/unit/sanitiser/_source.js

@@ -0,0 +1,114 @@
+var sanitize = require( '../../../sanitiser/_source' );
+
+var success_response = { error: false };
+
+module.exports.tests = {};
+
+module.exports.tests.no_sources = function(test, common) {
+  test('source is not set', function(t) {
+    var req = {
+      query: { }
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
+    t.deepEqual(response, success_response, 'no error returned');
+    t.end();
+  });
+
+  test('source is empty string', function(t) {
+    var req = {
+      query: {
+        source: ''
+      }
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(req.clean.types, {}, 'clean.types should be empty object');
+    t.deepEqual(response, success_response, 'no error returned');
+    t.end();
+  });
+};
+
+module.exports.tests.valid_sources = function(test, common) {
+  test('geonames source', function(t) {
+    var req = {
+      query: {
+        source: 'geonames'
+      }
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(req.clean.types, { from_source: ['geoname'] }, 'clean.types should contain from_source entry with geonames');
+    t.deepEqual(response, success_response, 'no error returned');
+    t.end();
+  });
+
+  test('openstreetmap source', function(t) {
+    var req = {
+      query: {
+        source: 'openstreetmap'
+      }
+    };
+    var expected_types = {
+      from_source: ['osmaddress', 'osmnode', 'osmway']
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(req.clean.types, expected_types, 'clean.types should contain from_source entry with multiple entries for openstreetmap');
+    t.deepEqual(response, success_response, 'no error returned');
+    t.end();
+  });
+
+  test('multiple sources', function(t) {
+    var req = {
+      query: {
+        source: 'openstreetmap,openaddresses'
+      }
+    };
+    var expected_types = {
+      from_source: ['osmaddress', 'osmnode', 'osmway', 'openaddresses']
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(req.clean.types, expected_types,
+                'clean.types should contain from_source entry with multiple entries for openstreetmap and openadresses');
+    t.deepEqual(response, success_response, 'no error returned');
+    t.end();
+  });
+};
+
+module.exports.tests.invalid_sources = function(test, common) {
+  test('geonames source', function(t) {
+    var req = {
+      query: {
+        source: 'notasource'
+      }
+    };
+    var expected_response = {
+      error: true,
+      msg: '`notasource` is an invalid source parameter. Valid options: geonames, openaddresses, quattroshapes, openstreetmap'
+    };
+
+    var response = sanitize(req);
+
+    t.deepEqual(response, expected_response, 'error with message returned');
+    t.deepEqual(req.clean.types, { }, 'clean.types should remain empty');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANTIZE _source ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/place.js

@@ -2,14 +2,14 @@
 var place  = require('../../../sanitiser/place'),
     _sanitize = place.sanitize,
     middleware = place.middleware,
-    indeces = require('../../../query/indeces'),
+    types = require('../../../query/types'),
     delimiter = ':',
     defaultLengthError = function(input) { return 'invalid param \''+ input + '\': text length, must be >0'; },
     defaultFormatError = 'invalid: must be of the format type:id for ex: \'geoname:4163334\'',
     defaultError = 'invalid param \'id\': text length, must be >0',
     defaultMissingTypeError = function(input) {
       var type = input.split(delimiter)[0];
-      return type + ' is invalid. It must be one of these values - [' + indeces.join(', ') + ']'; },
+      return type + ' is invalid. It must be one of these values - [' + types.join(', ') + ']'; },
     defaultClean = { ids: [ { id: '123', type: 'geoname' } ], details: true },
     sanitize = function(query, cb) { _sanitize({'query':query}, cb); },
     inputs = {

test/unit/sanitiser/reverse.js

@@ -5,12 +5,11 @@ var suggest  = require('../../../sanitiser/reverse'),
     delim = ',',
     defaultError = 'missing param \'lat\'',
     defaultClean =  { lat:0,
-                      layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 
-                                'locality', 'local_admin', 'osmaddress', 'openaddresses' ], 
+                      types: {
+                      },
                       lon: 0,
                       size: 10,
                       details: true,
-                      default_layers_set: true,
                       categories: []
                     },
     sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
@@ -158,7 +157,7 @@ module.exports.tests.sanitize_details = function(test, common) {
 module.exports.tests.sanitize_layers = function(test, common) {
   test('unspecified', function(t) {
     sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
+      t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
       t.end();
     });
   });
@@ -173,21 +172,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
   test('poi (alias) layer', function(t) {
     var poi_layers = ['geoname','osmnode','osmway'];
     sanitize({ layers: 'poi', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, poi_layers, 'poi layers set');
+      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
       t.end();
     });
   });
   test('admin (alias) layer', function(t) {
     var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
     sanitize({ layers: 'admin', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, admin_layers, 'admin layers set');
+      t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
       t.end();
     });
   });
   test('address (alias) layer', function(t) {
     var address_layers = ['osmaddress','openaddresses'];
     sanitize({ layers: 'address', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, address_layers, 'address layers set');
+      t.deepEqual(clean.types.from_layers, address_layers, 'address layers set');
       t.end();
     });
   });
@@ -195,7 +194,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var poi_layers = ['geoname','osmnode','osmway'];
     var reg_layers = ['admin0', 'admin1'];
     sanitize({ layers: 'poi,admin0,admin1', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(poi_layers), 'poi + regular layers');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
       t.end();
     });
   });
@@ -203,7 +202,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
     var reg_layers   = ['geoname', 'osmway'];
     sanitize({ layers: 'admin,geoname,osmway', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
       t.end();
     });
   });
@@ -211,21 +210,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var address_layers = ['osmaddress','openaddresses'];
     var reg_layers   = ['geoname', 'osmway'];
     sanitize({ layers: 'address,geoname,osmway', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(address_layers), 'address + regular layers set');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
       t.end();
     });
   });
   test('alias layer plus regular layers (no duplicates)', function(t) {
     var poi_layers = ['geoname','osmnode','osmway'];
     sanitize({ layers: 'poi,geoname,osmnode', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, poi_layers, 'poi layers found (no duplicates)');
+      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
       t.end();
     });
   });
   test('multiple alias layers (no duplicates)', function(t) {
     var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
     sanitize({ layers: 'poi,admin', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, alias_layers, 'all layers found (no duplicates)');
+      t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
       t.end();
     });
   });

test/unit/sanitiser/search.js

@@ -8,12 +8,11 @@ var search  = require('../../../sanitiser/search'),
     delim = ',',
     defaultError = 'invalid param \'input\': text length, must be >0',
     defaultClean =  { input: 'test', 
-                      layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood', 
-                                'locality', 'local_admin', 'osmaddress', 'openaddresses' ], 
+                      types: {
+                      },
                       size: 10,
                       details: true,
                       parsed_input: defaultParsed,
-                      default_layers_set: true
                     },
     sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
 
@@ -32,13 +31,10 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-module.exports.tests.sanitize_input = function(test, common) {
-  var inputs = {
-    invalid: [ '', 100, null, undefined, new Date() ],
-    valid: [ 'a', 'aa', 'aaaaaaaa' ]
-  };
+module.exports.tests.sanitize_invalid_input = function(test, common) {
   test('invalid input', function(t) {  
-    inputs.invalid.forEach( function( input ){
+    var invalid = [ '', 100, null, undefined, new Date() ];
+    invalid.forEach( function( input ){
       sanitize({ input: input }, function( err, clean ){
         t.equal(err, 'invalid param \'input\': text length, must be >0', input + ' is an invalid input');
         t.equal(clean, undefined, 'clean not set');
@@ -46,16 +42,26 @@ module.exports.tests.sanitize_input = function(test, common) {
     });
     t.end();
   });
-  test('valid input', function(t) {  
-    inputs.valid.forEach( function( input ){
-      sanitize({ input: input }, function( err, clean ){
-        var expected = JSON.parse(JSON.stringify( defaultClean ));
-        expected.input = input;
-        expected.parsed_input.target_layer = _input.getTargetLayers(input);
+};
 
-        t.equal(err, undefined, 'no error');
-        t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
-      });
+module.exports.tests.sanitise_valid_input = function(test, common) {
+  test('valid short input', function(t) {
+    sanitize({ input: 'a' }, function( err, clean ){
+      t.equal(err, undefined, 'no error');
+    });
+    t.end();
+  });
+
+  test('valid not-quite-as-short input', function(t) {
+    sanitize({ input: 'aa' }, function( err, clean ){
+      t.equal(err, undefined, 'no error');
+    });
+    t.end();
+  });
+
+  test('valid longer input', function(t) {
+    sanitize({ input: 'aaaaaaaa' }, function( err, clean ){
+      t.equal(err, undefined, 'no error');
     });
     t.end();
   });
@@ -70,11 +76,9 @@ module.exports.tests.sanitize_input_with_delim = function(test, common) {
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.input = input;
 
-        expected.parsed_input = parser(input);
+        expected.parsed_input = parser.get_parsed_address(input);
         t.equal(err, undefined, 'no error');
         t.equal(clean.parsed_input.name, expected.parsed_input.name, 'clean name set correctly');
-        t.equal(clean.parsed_input.admin_parts, expected.parsed_input.admin_parts, 'clean admin_parts set correctly');
-        t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
 
       });
     });
@@ -101,10 +105,8 @@ module.exports.tests.sanitize_lat = function(test, common) {
       sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.lat = parseFloat( lat );
-        expected.lon = 0;
         t.equal(err, undefined, 'no error');
-        expected.parsed_input = parser('test');
-        t.deepEqual(clean, expected, 'clean set correctly (' + lat + ')');
+        t.deepEqual(clean.lat, expected.lat, 'clean lat set correctly (' + lat + ')');
       });
     });
     t.end();
@@ -120,10 +122,8 @@ module.exports.tests.sanitize_lon = function(test, common) {
       sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.lon = parseFloat( lon );
-        expected.lat = 0;
         t.equal(err, undefined, 'no error');
-        expected.parsed_input = parser('test');
-        t.deepEqual(clean, expected, 'clean set correctly (' + lon + ')');
+        t.deepEqual(clean.lon, expected.lon, 'clean set correctly (' + lon + ')');
       });
     });
     t.end();
@@ -133,32 +133,25 @@ module.exports.tests.sanitize_lon = function(test, common) {
 module.exports.tests.sanitize_optional_geo = function(test, common) {
   test('no lat/lon', function(t) {
     sanitize({ input: 'test' }, function( err, clean ){
-      var expected = defaultClean;
       t.equal(err, undefined, 'no error');
       t.equal(clean.lat, undefined, 'clean set without lat');
       t.equal(clean.lon, undefined, 'clean set without lon');
-      expected.parsed_input = parser('test');
-      t.deepEqual(clean, expected, 'clean set without lat/lon');
     });
     t.end();
   });
   test('no lat', function(t) {
     sanitize({ input: 'test', lon: 0 }, function( err, clean ){
-      var expected = JSON.parse(JSON.stringify( defaultClean ));
-      expected.lon = 0;
+      var expected_lon = 0;
       t.equal(err, undefined, 'no error');
-      expected.parsed_input = parser('test');
-      t.deepEqual(clean, expected, 'clean set correctly (without any lat)');
+      t.deepEqual(clean.lon, expected_lon, 'clean set correctly (without any lat)');
     });
     t.end();
   });
   test('no lon', function(t) {
     sanitize({ input: 'test', lat: 0 }, function( err, clean ){
-      var expected = JSON.parse(JSON.stringify( defaultClean ));
-      expected.lat = 0;
+      var expected_lat = 0;
       t.equal(err, undefined, 'no error');
-      expected.parsed_input = parser('test');
-      t.deepEqual(clean, expected, 'clean set correctly (without any lon)');
+      t.deepEqual(clean.lat, expected_lat, 'clean set correctly (without any lon)');
     });
     t.end();
   });
@@ -166,8 +159,6 @@ module.exports.tests.sanitize_optional_geo = function(test, common) {
 
 module.exports.tests.sanitize_bbox = function(test, common) {
   var bboxes = {
-    invalid_coordinates: [
-    ],
     invalid: [
       '91;-181,-91,181', // invalid - semicolon between coordinates
       'these,are,not,numbers',
@@ -194,22 +185,11 @@ module.exports.tests.sanitize_bbox = function(test, common) {
     ]
     
   };
-  test('invalid bbox coordinates', function(t) {  
-    bboxes.invalid_coordinates.forEach( function( bbox ){
-      sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
-        t.ok(err.match(/Invalid (lat|lon)/), bbox + ' is invalid');
-        t.equal(clean, undefined, 'clean not set');
-      });
-    });
-    t.end();
-  });
   test('invalid bbox', function(t) {  
     bboxes.invalid.forEach( function( bbox ){
       sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
-        var expected = JSON.parse(JSON.stringify( defaultClean ));
         t.equal(err, undefined, 'no error');
-        expected.parsed_input = parser('test');
-        t.deepEqual(clean, expected, 'falling back on 50km distance from centroid');
+        t.equal(clean.bbox, undefined, 'falling back on 50km distance from centroid');
       });
     });
     t.end();
@@ -217,19 +197,17 @@ module.exports.tests.sanitize_bbox = function(test, common) {
   test('valid bbox', function(t) {  
     bboxes.valid.forEach( function( bbox ){
       sanitize({ input: 'test', bbox: bbox }, function( err, clean ){
-        var expected = JSON.parse(JSON.stringify( defaultClean ));
         var bboxArray = bbox.split(',').map(function(i) {
           return parseInt(i);
         });
-        expected.bbox = {
+        var expected_bbox = {
           right: Math.max(bboxArray[0], bboxArray[2]),
           top: Math.max(bboxArray[1], bboxArray[3]),
           left: Math.min(bboxArray[0], bboxArray[2]),
           bottom: Math.min(bboxArray[1], bboxArray[3])
         };
         t.equal(err, undefined, 'no error');
-        expected.parsed_input = parser('test');
-        t.deepEqual(clean, expected, 'clean set correctly (' + bbox + ')');
+        t.deepEqual(clean.bbox, expected_bbox, 'clean set correctly (' + bbox + ')');
       });
     });
     t.end();
@@ -320,7 +298,7 @@ module.exports.tests.sanitize_details = function(test, common) {
 module.exports.tests.sanitize_layers = function(test, common) {
   test('unspecified', function(t) {
     sanitize({ layers: undefined, input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
+      t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
       t.end();
     });
   });
@@ -335,21 +313,22 @@ module.exports.tests.sanitize_layers = function(test, common) {
   test('poi (alias) layer', function(t) {
     var poi_layers = ['geoname','osmnode','osmway'];
     sanitize({ layers: 'poi', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, poi_layers, 'poi layers set');
+      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers set');
       t.end();
     });
   });
   test('admin (alias) layer', function(t) {
     var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
     sanitize({ layers: 'admin', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, admin_layers, 'admin layers set');
+      t.deepEqual(clean.types.from_layers, admin_layers, 'admin layers set');
       t.end();
     });
   });
   test('address (alias) layer', function(t) {
     var address_layers = ['osmaddress','openaddresses'];
     sanitize({ layers: 'address', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, address_layers, 'address layers set');
+      t.deepEqual(clean.types.from_layers, address_layers, 'types from layers set');
+      t.deepEqual(clean.types.from_address_parser, _input.allLayers, 'address parser uses default layers');
       t.end();
     });
   });
@@ -357,7 +336,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var poi_layers = ['geoname','osmnode','osmway'];
     var reg_layers = ['admin0', 'admin1'];
     sanitize({ layers: 'poi,admin0,admin1', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(poi_layers), 'poi + regular layers');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(poi_layers), 'poi + regular layers');
       t.end();
     });
   });
@@ -365,7 +344,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var admin_layers = ['admin0','admin1','admin2','neighborhood','locality','local_admin'];
     var reg_layers   = ['geoname', 'osmway'];
     sanitize({ layers: 'admin,geoname,osmway', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(admin_layers), 'admin + regular layers set');
       t.end();
     });
   });
@@ -373,21 +352,21 @@ module.exports.tests.sanitize_layers = function(test, common) {
     var address_layers = ['osmaddress','openaddresses'];
     var reg_layers   = ['geoname', 'osmway'];
     sanitize({ layers: 'address,geoname,osmway', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, reg_layers.concat(address_layers), 'address + regular layers set');
+      t.deepEqual(clean.types.from_layers, reg_layers.concat(address_layers), 'address + regular layers set');
       t.end();
     });
   });
   test('alias layer plus regular layers (no duplicates)', function(t) {
     var poi_layers = ['geoname','osmnode','osmway'];
     sanitize({ layers: 'poi,geoname,osmnode', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, poi_layers, 'poi layers found (no duplicates)');
+      t.deepEqual(clean.types.from_layers, poi_layers, 'poi layers found (no duplicates)');
       t.end();
     });
   });
   test('multiple alias layers (no duplicates)', function(t) {
     var alias_layers = ['geoname','osmnode','osmway','admin0','admin1','admin2','neighborhood','locality','local_admin'];
     sanitize({ layers: 'poi,admin', input: 'test' }, function( err, clean ){
-      t.deepEqual(clean.layers, alias_layers, 'all layers found (no duplicates)');
+      t.deepEqual(clean.types.from_layers, alias_layers, 'all layers found (no duplicates)');
       t.end();
     });
   });
@@ -420,8 +399,6 @@ module.exports.tests.middleware_success = function(test, common) {
     var req = { query: { input: 'test' }};
     var next = function( message ){
       t.equal(message, undefined, 'no error message set');
-      req.clean.parsed_input = parser('test');
-      t.deepEqual(req.clean, defaultClean);
       t.end();
     };
     middleware( req, undefined, next );

test/unit/sanitiser/suggest.js

@@ -15,7 +15,6 @@ var suggest  = require('../../../sanitiser/suggest'),
                       lat:0,
                       lon:0,
                       parsed_input: defaultParsed,
-                      default_layers_set: true
                     },
     sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
 
@@ -272,7 +271,7 @@ module.exports.tests.sanitize_details = function(test, common) {
 module.exports.tests.sanitize_layers = function(test, common) {
   test('unspecified', function(t) {
     sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      t.deepEqual(clean.layers, defaultClean.layers, 'default layers set');
+      t.deepEqual(clean.types.from_layers, defaultClean.types.from_layers, 'default layers set');
       t.end();
     });
   });