feat: Add categories filter param and /nearby endpoint

Merge pull request #546 from pelias/categories

helper/geojsonify.js

@@ -1,66 +1,18 @@
 
-var GeoJSON = require('geojson'),
+var GeoJSON = require('geojson');
-    extent = require('geojson-extent'),
+var extent = require('geojson-extent');
-    labelGenerator = require('./labelGenerator'),
+var labelGenerator = require('./labelGenerator');
-    logger = require('pelias-logger').get('api'),
+var logger = require('pelias-logger').get('api');
-    type_mapping = require('./type_mapping'),
+var type_mapping = require('./type_mapping');
-    Document = require('pelias-model').Document,
+var _ = require('lodash');
-    _ = require('lodash');
+var addDetails = require('./geojsonify_place_details');
+var addMetaData = require('./geojsonify_meta_data');
 
-// Properties to be copied
+function geojsonifyPlaces( params, docs ){
-var DETAILS_PROPS = [
-  'housenumber',
-  'street',
-  'postalcode',
-  'confidence',
-  'distance',
-  'country',
-  'country_gid',
-  'country_a',
-  'macroregion',
-  'macroregion_gid',
-  'macroregion_a',
-  'region',
-  'region_gid',
-  'region_a',
-  'macrocounty',
-  'macrocounty_gid',
-  'macrocounty_a',
-  'county',
-  'county_gid',
-  'county_a',
-  'localadmin',
-  'localadmin_gid',
-  'localadmin_a',
-  'locality',
-  'locality_gid',
-  'locality_a',
-  'borough',
-  'borough_gid',
-  'borough_a',
-  'neighbourhood',
-  'neighbourhood_gid',
-  'bounding_box'
-];
-
-
-function lookupSource(src) {
-  return src.source;
-}
-
-function lookupSourceId(src) {
-  return src.source_id;
-}
-
-function lookupLayer(src) {
-  return src.layer;
-}
-
-function geojsonifyPlaces( docs ){
 
   // flatten & expand data for geojson conversion
   var geodata = docs
-    .map(geojsonifyPlace)
+    .map(geojsonifyPlace.bind(null, params))
     .filter( function( doc ){
       return !!doc;
     });
@@ -83,7 +35,7 @@ function geojsonifyPlaces( docs ){
   return geojson;
 }
 
-function geojsonifyPlace(place) {
+function geojsonifyPlace(params, place) {
 
   // something went very wrong
   if( !place || !place.hasOwnProperty( 'center_point' ) ) {
@@ -93,7 +45,8 @@ function geojsonifyPlace(place) {
   var output = {};
 
   addMetaData(place, output);
-  addDetails(place, output);
+  addName(place, output);
+  addDetails(params, place, output);
   addLabel(place, output);
 
 
@@ -106,17 +59,15 @@ function geojsonifyPlace(place) {
 }
 
 /**
- * Add details properties
+ * Validate and add name property
  *
  * @param {object} src
  * @param {object} dst
  */
-function addDetails(src, dst) {
+function addName(src, dst) {
   // map name
   if( !src.name || !src.name.default ) { return warning(src); }
   dst.name = src.name.default;
-
-  copyProperties(src, DETAILS_PROPS, dst);
 }
 
 /**
@@ -206,65 +157,6 @@ function computeBBox(geojson, geojsonExtentPoints) {
   }
 }
 
-/**
- * Copy specified properties from source to dest.
- * Ignore missing properties.
- *
- * @param {object} source
- * @param {[]} props
- * @param {object} dst
- */
-function copyProperties( source, props, dst ) {
-  props.forEach( function ( prop ) {
-
-    if ( source.hasOwnProperty( prop ) ) {
-
-      // array value, take first item in array (at this time only used for admin values)
-      if (source[prop] instanceof Array) {
-        if (source[prop].length === 0) {
-          return;
-        }
-        if (source[prop][0]) {
-          dst[prop] = source[prop][0];
-        }
-      }
-
-      // simple value
-      else {
-        dst[prop] = source[prop];
-      }
-    }
-  });
-}
-
-/**
- * Create a gid from a document
- * @TODO modify all importers to create separate source and layer fields to remove mapping
- *
- * @param {object} src
- */
-function makeGid(src) {
-  var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
-  return doc.getGid();
-}
-
-/**
- * Determine and set place id, type, and source
- *
- * @param {object} src
- * @param {object} dst
- */
-function addMetaData(src, dst) {
-  dst.id = src._id;
-  dst.gid = makeGid(src);
-  dst.layer = lookupLayer(src);
-  dst.source = lookupSource(src);
-  dst.source_id = lookupSourceId(src);
-  if (src.hasOwnProperty('bounding_box')) {
-    dst.bounding_box = src.bounding_box;
-  }
-}
-
 /**
  * emit a warning if the doc format is invalid
  *
@@ -276,4 +168,4 @@ function warning( doc ) {
 }
 
 
-module.exports.search = geojsonifyPlaces;
+module.exports = geojsonifyPlaces;

helper/geojsonify_meta_data.js

@@ -0,0 +1,42 @@
+var Document = require('pelias-model').Document;
+
+/**
+ * Determine and set place id, type, and source
+ *
+ * @param {object} src
+ * @param {object} dst
+ */
+function addMetaData(src, dst) {
+  dst.id = src._id;
+  dst.gid = makeGid(src);
+  dst.layer = lookupLayer(src);
+  dst.source = lookupSource(src);
+  dst.source_id = lookupSourceId(src);
+  if (src.hasOwnProperty('bounding_box')) {
+    dst.bounding_box = src.bounding_box;
+  }
+}
+
+/**
+ * Create a gid from a document
+ *
+ * @param {object} src
+ */
+function makeGid(src) {
+  var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
+  return doc.getGid();
+}
+
+function lookupSource(src) {
+  return src.source;
+}
+
+function lookupSourceId(src) {
+  return src.source_id;
+}
+
+function lookupLayer(src) {
+  return src.layer;
+}
+
+module.exports = addMetaData;

helper/geojsonify_place_details.js

@@ -0,0 +1,133 @@
+var _ = require('lodash');
+
+// Properties to be copied
+// If a property is identified as a single string, assume it should be presented as a string in response
+// If something other than string is desired, use the following structure: { name: 'category', type: 'array' }
+var DETAILS_PROPS = [
+  { name: 'housenumber',       type: 'string' },
+  { name: 'street',            type: 'string' },
+  { name: 'postalcode',        type: 'string' },
+  { name: 'confidence',        type: 'default' },
+  { name: 'distance',          type: 'default' },
+  { name: 'country',           type: 'string' },
+  { name: 'country_gid',       type: 'string' },
+  { name: 'country_a',         type: 'string' },
+  { name: 'macroregion',       type: 'string' },
+  { name: 'macroregion_gid',   type: 'string' },
+  { name: 'macroregion_a',     type: 'string' },
+  { name: 'region',            type: 'string' },
+  { name: 'region_gid',        type: 'string' },
+  { name: 'region_a',          type: 'string' },
+  { name: 'macrocounty',       type: 'string' },
+  { name: 'macrocounty_gid',   type: 'string' },
+  { name: 'macrocounty_a',     type: 'string' },
+  { name: 'county',            type: 'string' },
+  { name: 'county_gid',        type: 'string' },
+  { name: 'county_a',          type: 'string' },
+  { name: 'localadmin',        type: 'string' },
+  { name: 'localadmin_gid',    type: 'string' },
+  { name: 'localadmin_a',      type: 'string' },
+  { name: 'locality',          type: 'string' },
+  { name: 'locality_gid',      type: 'string' },
+  { name: 'locality_a',        type: 'string' },
+  { name: 'borough',           type: 'string' },
+  { name: 'borough_gid',       type: 'string' },
+  { name: 'borough_a',         type: 'string' },
+  { name: 'neighbourhood',     type: 'string' },
+  { name: 'neighbourhood_gid', type: 'string' },
+  { name: 'bounding_box',      type: 'default' },
+  { name: 'category',          type: 'array',     condition: checkCategoryParam }
+];
+
+function checkCategoryParam(params) {
+  return _.isObject(params) && params.hasOwnProperty('categories');
+}
+
+/**
+ * Add details properties
+ *
+ * @param {object} params clean query params
+ * @param {object} src
+ * @param {object} dst
+ */
+function addDetails(params, src, dst) {
+  copyProperties(params, src, DETAILS_PROPS, dst);
+}
+
+/**
+ * Copy specified properties from source to dest.
+ * Ignore missing properties.
+ *
+ * @param {object} params clean query params
+ * @param {object} source
+ * @param {[]} props
+ * @param {object} dst
+ */
+function copyProperties( params, source, props, dst ) {
+  props.forEach( function ( prop ) {
+
+    // if condition isn't met, just return without setting the property
+    if (_.isFunction(prop.condition) && !prop.condition(params)) {
+      return;
+    }
+
+    var property = {
+      name: prop.name || prop,
+      type: prop.type || 'default'
+    };
+
+    var value = null;
+    if ( source.hasOwnProperty( property.name ) ) {
+
+      switch (property.type) {
+        case 'string':
+          value = getStringValue(source[property.name]);
+          break;
+        case 'array':
+          value = getArrayValue(source[property.name]);
+          break;
+        // default behavior is to copy property exactly as is
+        default:
+          value = source[property.name];
+      }
+
+      if (_.isNumber(value) || (value && !_.isEmpty(value))) {
+        dst[property.name] = value;
+      }
+    }
+  });
+}
+
+function getStringValue(property) {
+  // isEmpty check works for all types of values: strings, arrays, objects
+  if (_.isEmpty(property)) {
+    return '';
+  }
+
+  if (_.isString(property)) {
+    return property;
+  }
+
+  // array value, take first item in array (at this time only used for admin values)
+  if (_.isArray(property)) {
+    return property[0];
+  }
+
+  return _.toString(property);
+}
+
+
+function getArrayValue(property) {
+  // isEmpty check works for all types of values: strings, arrays, objects
+  if (_.isEmpty(property)) {
+    return '';
+  }
+
+  if (_.isArray(property)) {
+    return property;
+  }
+
+  return [property];
+}
+
+module.exports = addDetails;

middleware/geocodeJSON.js

@@ -1,6 +1,6 @@
 var url = require('url');
 var extend = require('extend');
-var geojsonify = require('../helper/geojsonify').search;
+var geojsonify = require('../helper/geojsonify');
 
 /**
  * Returns a middleware function that converts elasticsearch
@@ -72,7 +72,7 @@ function convertToGeocodeJSON(req, res, next, opts) {
   res.body.geocoding.timestamp = new Date().getTime();
 
   // convert docs to geojson and merge with geocoding block
-  extend(res.body, geojsonify(res.data || []));
+  extend(res.body, geojsonify(req.clean, res.data || []));
 
   next();
 }

package.json

@@ -50,6 +50,7 @@
     "lodash": "^4.5.0",
     "markdown": "0.5.0",
     "morgan": "1.7.0",
+    "pelias-categories": "1.0.0",
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.1.0",

query/reverse.js

@@ -17,6 +17,7 @@ query.sort( peliasQuery.view.sort_distance );
 query.filter( peliasQuery.view.boundary_circle );
 query.filter( peliasQuery.view.sources );
 query.filter( peliasQuery.view.layers );
+query.filter( peliasQuery.view.categories );
 
 // --------------------------------
 
@@ -65,6 +66,11 @@ function generateQuery( clean ){
     });
   }
 
+  // categories
+  if (clean.categories) {
+    vs.var('input:categories', clean.categories);
+  }
+
   return query.render( vs );
 }
 

query/search.js

@@ -44,6 +44,8 @@ query.filter( peliasQuery.view.boundary_circle );
 query.filter( peliasQuery.view.boundary_rect );
 query.filter( peliasQuery.view.sources );
 query.filter( peliasQuery.view.layers );
+query.filter( peliasQuery.view.categories );
+
 // --------------------------------
 
 /**
@@ -63,6 +65,11 @@ function generateQuery( clean ){
   // layers
   vs.var( 'layers', clean.layers);
 
+  // categories
+  if (clean.categories) {
+    vs.var('input:categories', clean.categories);
+  }
+
   // size
   if( clean.querySize ) {
     vs.var( 'size', clean.querySize );

routes/v1.js

@@ -7,7 +7,8 @@ var sanitisers = {
   autocomplete: require('../sanitiser/autocomplete'),
   place: require('../sanitiser/place'),
   search: require('../sanitiser/search'),
-  reverse: require('../sanitiser/reverse')
+  reverse: require('../sanitiser/reverse'),
+  nearby: require('../sanitiser/nearby')
 };
 
 /** ----------------------- middleware ------------------------ **/
@@ -101,6 +102,22 @@ function addRoutes(app, peliasConfig) {
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
+    nearby: createRouter([
+      sanitisers.nearby.middleware,
+      middleware.calcSize(),
+      controllers.search(peliasConfig, undefined, reverseQuery),
+      postProc.distances('point.'),
+      // reverse confidence scoring depends on distance from origin
+      //  so it must be calculated first
+      postProc.confidenceScoresReverse(),
+      postProc.dedupe(),
+      postProc.localNamingConventions(),
+      postProc.renamePlacenames(),
+      postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
+      postProc.geocodeJSON(peliasConfig, base),
+      postProc.sendJSON
+    ]),
     place: createRouter([
       sanitisers.place.middleware,
       controllers.place(peliasConfig),
@@ -129,6 +146,7 @@ function addRoutes(app, peliasConfig) {
   app.get ( base + 'search',       routers.search );
   app.post( base + 'search',       routers.search );
   app.get ( base + 'reverse',      routers.reverse );
+  app.get ( base + 'nearby',       routers.nearby );
 
 }
 

sanitiser/_categories.js

@@ -1,34 +1,53 @@
 
 var check = require('check-types');
+var categoryTaxonomy = require('pelias-categories');
+
+var ERRORS = {
+  empty: 'Categories parameter cannot be left blank. See documentation of service for valid options.',
+  invalid: 'Invalid categories parameter value(s). See documentation of service for valid options.'
+};
 
 // validate inputs, convert types and apply defaults
-function sanitize( raw, clean ){
+function sanitize( raw, clean, categories ) {
+
+  categories = categories || categoryTaxonomy;
 
   // error & warning messages
-  var messages = { errors: [], warnings: [] };
+  var messages = {errors: [], warnings: []};
 
-  // default case (no categories specified in GET params)
+  // it's not a required parameter, so if it's not provided just move on
-  clean.categories = [];
+  if (!raw.hasOwnProperty('categories')) {
+    return messages;
+  }
 
-  // if categories string has been set
+  if (!check.nonEmptyString(raw.categories)) {
-  if( check.nonEmptyString( raw.categories ) ){
+    messages.errors.push(ERRORS.empty);
+    return messages;
+  }
 
-    // map input categories to valid format
+  // if categories string has been set
+  // map input categories to valid format
+  try {
     clean.categories = raw.categories.split(',')
       .map(function (cat) {
         return cat.toLowerCase().trim(); // lowercase inputs
       })
-      .filter( function( cat ) {
+      .filter(function (cat) {
-        return ( cat.length > 0 );
+        if (check.nonEmptyString(cat) && categories.isValidCategory(cat)) {
+          return true;
+        }
+        throw new Error('Empty string value');
       });
+  } catch (err) {
+    // remove everything from the list if there was any error
+    delete clean.categories;
+  }
 
-    if( !clean.categories.length ){
+  if (check.undefined(clean.categories) || check.emptyArray(clean.categories)) {
-      messages.warnings.push( 'invalid \'categories\': no valid category strings found');
+    messages.errors.push(ERRORS.invalid);
-    }
   }
 
   return messages;
-
 }
 
 // export function

sanitiser/autocomplete.js

@@ -12,6 +12,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       sources_and_layers: require('../sanitiser/_sources_and_layers'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_autocomplete: require('../sanitiser/_geo_autocomplete'),
+      categories: require('../sanitiser/_categories')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };

sanitiser/nearby.js

@@ -0,0 +1,21 @@
+var _ = require('lodash');
+var sanitizeAll = require('../sanitiser/sanitizeAll');
+var reverseSanitizers = require('./reverse').sanitiser_list;
+
+// add categories to the sanitizer list
+var sanitizers = _.merge({}, reverseSanitizers, {
+  categories: require('../sanitiser/_categories')
+});
+
+var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+
+// export sanitize for testing
+module.exports.sanitize = sanitize;
+module.exports.sanitiser_list = sanitizers;
+
+// middleware
+module.exports.middleware = function( req, res, next ){
+  sanitize( req, function( err, clean ){
+    next();
+  });
+};

sanitiser/reverse.js

@@ -11,7 +11,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       size: require('../sanitiser/_size')(/* use defaults*/),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_reverse: require('../sanitiser/_geo_reverse'),
-      boundary_country: require('../sanitiser/_boundary_country'),
+      boundary_country: require('../sanitiser/_boundary_country')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };

sanitiser/search.js

@@ -13,6 +13,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),
       boundary_country: require('../sanitiser/_boundary_country'),
+      categories: require('../sanitiser/_categories')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };

test/unit/fixture/search_with_category_filtering.js

@@ -0,0 +1,86 @@
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [{
+        'terms': {
+          'category': ['retail', 'food']
+        }
+      }]
+    }
+  },
+  'size': 20,
+  'track_scores': true,
+  'sort': [
+    '_score'
+  ]
+};

test/unit/helper/geojsonify.js

@@ -4,9 +4,9 @@ var geojsonify = require('../../../helper/geojsonify');
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
-  test('valid interface .search()', function(t) {
+  test('valid interface', function(t) {
-    t.equal(typeof geojsonify.search, 'function', 'search is a function');
+    t.equal(typeof geojsonify, 'function', 'search is a function');
-    t.equal(geojsonify.search.length, 1, 'accepts x arguments');
+    t.equal(geojsonify.length, 2, 'accepts x arguments');
     t.end();
   });
 };
@@ -30,14 +30,14 @@ module.exports.tests.earth = function(test, common) {
 
   test('earth', function(t) {
     t.doesNotThrow(function(){
-      geojsonify.search( earth );
+      geojsonify( {}, earth );
     });
     t.end();
   });
 
 };
 
-module.exports.tests.search = function(test, common) {
+module.exports.tests.geojsonify = function(test, common) {
 
   var input = [
     {
@@ -153,7 +153,11 @@ module.exports.tests.search = function(test, common) {
           'neighbourhood': 'test3',
           'housenumber': '13',
           'street': 'Liverpool Road',
-          'postalcode': 'N1 0RW'
+          'postalcode': 'N1 0RW',
+          'category': [
+            'food',
+            'nightlife'
+          ]
         }
       },
       {
@@ -208,14 +212,18 @@ module.exports.tests.search = function(test, common) {
           'county': 'New York',
           'borough': 'Manhattan',
           'locality': 'New York',
-          'neighbourhood': 'Koreatown'
+          'neighbourhood': 'Koreatown',
+          'category': [
+            'tourism',
+            'transport'
+          ]
         }
       }
     ]
   };
 
-  test('geojsonify.search(doc)', function(t) {
+  test('geojsonify(doc)', function(t) {
-    var json = geojsonify.search( input );
+    var json = geojsonify( {categories: 'foo'}, input );
 
     t.deepEqual(json, expected, 'all docs mapped');
     t.end();
@@ -245,7 +253,7 @@ module.exports.tests.search = function(test, common) {
           'default': 'East New York'
         },
         'source_id': '85816607',
-        'category': [],
+        'category': ['government'],
         '_id': '85816607',
         '_type': 'neighbourhood',
         '_score': 21.434,
@@ -328,6 +336,7 @@ module.exports.tests.search = function(test, common) {
             'source': 'whosonfirst',
             'source_id': '85816607',
             'name': 'East New York',
+            'category': ['government'],
             'confidence': 0.888,
             'country': 'United States',
             'country_gid': '85633793',
@@ -361,7 +370,67 @@ module.exports.tests.search = function(test, common) {
       ]
     };
 
-    var json = geojsonify.search( input );
+    var json = geojsonify( {categories: 'foo'}, input );
+
+    t.deepEqual(json, expected, 'all wanted properties exposed');
+    t.end();
+  });
+};
+
+module.exports.tests.categories = function (test, common) {
+  test('only set category if categories filter was used', function (t) {
+    var input = [
+      {
+        '_id': '85816607',
+        'bounding_box': {
+          'min_lat': 40.6514712164,
+          'max_lat': 40.6737320588,
+          'min_lon': -73.8967895508,
+          'max_lon': -73.8665771484
+        },
+        'source': 'whosonfirst',
+        'layer': 'neighbourhood',
+        'center_point': {
+          'lon': -73.881319,
+          'lat': 40.663303
+        },
+        'name': {
+          'default': 'East New York'
+        },
+        'source_id': '85816607',
+        'category': ['government']
+      }
+    ];
+
+    var expected = {
+      'type': 'FeatureCollection',
+      'bbox': [-73.8967895508, 40.6514712164, -73.8665771484, 40.6737320588],
+      'features': [
+        {
+          'type': 'Feature',
+          'properties': {
+            'id': '85816607',
+            'gid': 'whosonfirst:neighbourhood:85816607',
+            'layer': 'neighbourhood',
+            'source': 'whosonfirst',
+            'source_id': '85816607',
+            'name': 'East New York',
+            'category': ['government'],
+            'label': 'East New York'
+          },
+          'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],
+          'geometry': {
+            'type': 'Point',
+            'coordinates': [
+              -73.881319,
+              40.663303
+            ]
+          }
+        }
+      ]
+    };
+
+    var json = geojsonify( {categories: 'foo'}, input );
 
     t.deepEqual(json, expected, 'all wanted properties exposed');
     t.end();

test/unit/query/search.js

@@ -159,6 +159,18 @@ module.exports.tests.query = function(test, common) {
     t.end();
   });
 
+  test('categories filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'categories': ['retail','food']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_category_filtering');
+
+    t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/run.js

@@ -51,6 +51,8 @@ var tests = [
   require('./sanitiser/_text'),
   require('./sanitiser/_tokenizer'),
   require('./sanitiser/_deprecate_quattroshapes'),
+  require('./sanitiser/_categories'),
+  require('./sanitiser/nearby'),
   require('./src/backend'),
   require('./sanitiser/autocomplete'),
   require('./sanitiser/place'),

test/unit/sanitiser/_categories.js

@@ -0,0 +1,170 @@
+var sanitize = require( '../../../sanitiser/_categories');
+
+module.exports.tests = {};
+
+module.exports.tests.no_categories = function(test, common) {
+  test('categories not set', function(t) {
+    var req = {
+      query: { },
+      clean: { }
+    };
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.equal(req.clean.categories, undefined, 'no categories should be defined');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+
+  test('categories is empty string', function(t) {
+    var req = {
+      query: {
+        categories: ''
+      },
+      clean: { }
+    };
+
+    var expected_error = 'Categories parameter cannot be left blank. See documentation of service for valid options.';
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.equal(req.clean.categories, undefined, 'no categories should be defined');
+    t.deepEqual(messages.errors.length, 1, 'error returned');
+    t.deepEqual(messages.errors[0], expected_error, 'error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+
+  test('categories is an array of empty strings', function(t) {
+    var req = {
+      query: {
+        categories: ',,'
+      },
+      clean: { }
+    };
+
+    var expected_error = 'Invalid categories parameter value(s). See documentation of service for valid options.';
+
+    var messages = sanitize(req.query, req.clean);
+
+    t.equal(req.clean.categories, undefined, 'no categories should be defined');
+    t.deepEqual(messages.errors.length, 1, 'error returned');
+    t.deepEqual(messages.errors[0], expected_error, 'error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+    t.end();
+  });
+};
+
+module.exports.tests.valid_categories = function(test, common) {
+  var isValidCategoryCalled = 0;
+  var validCategories = {
+    isValidCategory: function (cat) {
+      isValidCategoryCalled++;
+      return ['food','health','financial','education','government'].indexOf(cat) !== -1; }
+  };
+
+  test('single category', function(t) {
+    isValidCategoryCalled = 0;
+
+    var req = {
+      query: {
+        categories: 'food'
+      },
+      clean: { }
+    };
+
+    var messages = sanitize(req.query, req.clean, validCategories);
+
+    t.deepEqual(req.clean.categories, ['food'], 'categories should contain food');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+
+    t.equal(isValidCategoryCalled, 1);
+
+    t.end();
+  });
+
+  test('multiple categories', function(t) {
+    isValidCategoryCalled = 0;
+    var req = {
+      query: {
+        categories: 'food,health'
+      },
+      clean: { }
+    };
+    var expectedCategories = ['food', 'health'];
+
+    var messages = sanitize(req.query, req.clean, validCategories);
+
+    t.deepEqual(req.clean.categories, expectedCategories,
+                'clean.categories should be an array with proper values');
+    t.deepEqual(messages.errors, [], 'no error returned');
+    t.deepEqual(messages.warnings, [], 'no warnings returned');
+
+    t.equal(isValidCategoryCalled, expectedCategories.length);
+
+    t.end();
+  });
+};
+
+module.exports.tests.invalid_categories = function(test, common) {
+  var isValidCategoryCalled = 0;
+  var validCategories = {
+    isValidCategory: function (cat) {
+      isValidCategoryCalled++;
+      return ['food','health','financial','education','government'].indexOf(cat) !== -1; }
+  };
+
+  test('garbage category', function(t) {
+    var req = {
+      query: {
+        categories: 'barf'
+      },
+      clean: { }
+    };
+    var expected_messages = {
+      errors: [
+        'Invalid categories parameter value(s). See documentation of service for valid options.'
+      ],
+      warnings: []
+    };
+
+    var messages = sanitize(req.query, req.clean, validCategories);
+
+    t.deepEqual(messages, expected_messages, 'error with message returned');
+    t.equal(req.clean.categories, undefined, 'clean.categories should remain empty');
+    t.end();
+  });
+
+  test('all garbage categories', function(t) {
+    var req = {
+      query: {
+        categories: 'barf,bleh'
+      },
+      clean: { }
+    };
+    var expected_messages = {
+      errors: [
+        'Invalid categories parameter value(s). See documentation of service for valid options.'
+      ],
+      warnings: []
+    };
+
+    var messages = sanitize(req.query, req.clean, validCategories);
+
+    t.deepEqual(messages, expected_messages, 'error with message returned');
+    t.equal(req.clean.categories, undefined, 'clean.categories should remain empty');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANTIZE _categories ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/autocomplete.js

@@ -6,7 +6,7 @@ module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
     var expected = [
       'singleScalarParameters', 'text', 'tokenizer', 'size', 'layers', 'sources',
-      'sources_and_layers', 'private', 'geo_autocomplete'
+      'sources_and_layers', 'private', 'geo_autocomplete', 'categories'
     ];
     t.deepEqual(Object.keys(autocomplete.sanitiser_list), expected);
     t.end();

test/unit/sanitiser/nearby.js

@@ -0,0 +1,61 @@
+
+var nearby  = require('../../../sanitiser/nearby');
+var defaults = require('../../../query/reverse_defaults');
+var sanitize = nearby.sanitize;
+var middleware = nearby.middleware;
+
+var defaultClean =  { 'point.lat': 0,
+                      'point.lon': 0,
+                      'boundary.circle.lat': 0,
+                      'boundary.circle.lon': 0,
+                      'boundary.circle.radius': parseFloat(defaults['boundary:circle:radius']),
+                      size: 10,
+                      private: false
+                    };
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('sanitize interface', function(t) {
+    t.equal(typeof sanitize, 'function', 'sanitize is a function');
+    t.equal(sanitize.length, 2, 'sanitize interface');
+    t.end();
+  });
+  test('middleware interface', function(t) {
+    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(middleware.length, 3, 'sanitizee has a valid middleware');
+    t.end();
+  });
+};
+
+module.exports.tests.sanitisers = function(test, common) {
+  test('check sanitiser list', function (t) {
+    var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'layers',
+      'sources', 'sources_and_layers', 'size', 'private', 'geo_reverse', 'boundary_country', 'categories'];
+    t.deepEqual(Object.keys(nearby.sanitiser_list), expected);
+    t.end();
+  });
+};
+
+module.exports.tests.middleware_success = function(test, common) {
+  test('middleware success', function(t) {
+    var req = { query: { 'point.lat': 0, 'point.lon': 0 }};
+    var next = function(){
+      t.deepEqual(req.errors, [], 'no error message set');
+      t.deepEqual(req.clean, defaultClean);
+      t.end();
+    };
+    middleware( req, undefined, next );
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SANTIZE /nearby ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/search.js

@@ -25,7 +25,7 @@ module.exports.tests.interface = function(test, common) {
 module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
     var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'text', 'size',
-      'layers', 'sources', 'sources_and_layers', 'private', 'geo_search', 'boundary_country' ];
+      'layers', 'sources', 'sources_and_layers', 'private', 'geo_search', 'boundary_country', 'categories' ];
     t.deepEqual(Object.keys(search.sanitiser_list), expected);
     t.end();
   });