Merge branch 'master' into multiple_suggesters

app.js

@@ -11,6 +11,7 @@ app.use( require('./middleware/jsonp') );
 /** ----------------------- sanitisers ----------------------- **/
 
 var sanitisers = {};
+sanitisers.doc      = require('./sanitiser/doc');
 sanitisers.suggest  = require('./sanitiser/suggest');
 sanitisers.search   = sanitisers.suggest;
 sanitisers.reverse  = require('./sanitiser/reverse');
@@ -19,6 +20,7 @@ sanitisers.reverse  = require('./sanitiser/reverse');
 
 var controllers     = {};
 controllers.index   = require('./controller/index');
+controllers.doc     = require('./controller/doc');
 controllers.suggest = require('./controller/suggest');
 controllers.suggest_poi = require('./controller/suggest_poi');
 controllers.suggest_admin = require('./controller/suggest_admin');
@@ -29,6 +31,9 @@ controllers.search  = require('./controller/search');
 // api root
 app.get( '/', controllers.index() );
 
+// doc API
+app.get( '/doc', sanitisers.doc.middleware, controllers.doc() );
+
 // suggest API
 app.get( '/suggest', sanitisers.suggest.middleware, controllers.suggest() );
 app.get( '/suggest/poi', sanitisers.suggest.middleware, controllers.suggest_poi() );

controller/doc.js

@@ -0,0 +1,55 @@
+
+var geojsonify = require('../helper/geojsonify').search;
+
+function setup( backend ){
+
+  // allow overriding of dependencies
+  backend = backend || require('../src/backend');
+  backend = new backend(); 
+  
+  function controller( req, res, next ){
+    
+    var docs = req.clean.ids.map( function(id) {
+      return {
+        _index: 'pelias',
+        _type: id.type,
+        _id: id.id
+      };
+    });
+
+    // backend command
+    var cmd = {
+      body: {
+        docs: docs
+      }
+    };
+    
+    // query new backend
+    backend.client.mget( cmd, function( err, data ){
+      
+      var docs = [];
+      // handle backend errors
+      if( err ){ return next( err ); }
+
+      if( data && data.docs && Array.isArray(data.docs) && data.docs.length ){
+        docs = data.docs.map( function( doc ){
+          return doc._source;
+        });
+      }
+
+      // convert docs to geojson
+      var geojson = geojsonify( docs );
+
+      // response envelope
+      geojson.date = new Date().getTime();
+
+      // respond
+      return res.status(200).json( geojson );
+    });
+
+  }
+
+  return controller;
+}
+
+module.exports = setup;

docs/get/mget.md

@@ -0,0 +1,97 @@
+# valid get query
+
+*Generated: Thu Oct 16 2014 17:02:52 GMT-0400 (EDT)*
+## Request
+```javascript
+{
+  "protocol": "http:",
+  "host": "localhost",
+  "method": "GET",
+  "port": 3100,
+  "path": "/get?id=geoname:4221195&id=geoname:4193595"
+}
+```
+
+## Response
+```javascript
+Status: 200
+{
+  "x-powered-by": "mapzen",
+  "charset": "utf8",
+  "cache-control": "public,max-age=60",
+  "server": "Pelias/0.0.0",
+  "access-control-allow-origin": "*",
+  "access-control-allow-methods": "GET",
+  "access-control-allow-headers": "X-Requested-With,content-type",
+  "access-control-allow-credentials": "true",
+  "content-type": "application/json; charset=utf-8",
+  "content-length": "567",
+  "etag": "W/\"237-e0425e36\"",
+  "date": "Thu, 16 Oct 2014 21:02:52 GMT",
+  "connection": "close"
+}
+```
+```javascript
+{
+  "type": "FeatureCollection",
+  "features": [
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -82.9207,
+          34.36094
+        ]
+      },
+      "properties": {
+        "name": "Sanders Grove Cemetery",
+        "admin0": "United States",
+        "admin1": "Georgia",
+        "admin2": "Hart County",
+        "text": "Sanders Grove Cemetery, Hart County, United States"
+      }
+    },
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -83.94213,
+          33.32262
+        ]
+      },
+      "properties": {
+        "name": "Etheredge Cemetery",
+        "admin0": "United States",
+        "admin1": "Georgia",
+        "admin2": "Butts County",
+        "text": "Etheredge Cemetery, Butts County, United States"
+      }
+    }
+  ],
+  "date": 1413493372681
+}
+```
+
+## Tests
+
+### ✓ valid response
+```javascript
+now = new Date().getTime()
+should.exist json
+should.not.exist json.error
+json.date.should.be.within now-5000, now+5000
+```
+
+### ✓ valid geojson
+```javascript
+json.type.should.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+```
+
+### ✓ 200 ok
+```javascript
+response.statusCode.should.equal 200
+```
+

docs/get/success.md

@@ -0,0 +1,80 @@
+# valid get query
+
+*Generated: Thu Oct 16 2014 17:02:52 GMT-0400 (EDT)*
+## Request
+```javascript
+{
+  "protocol": "http:",
+  "host": "localhost",
+  "method": "GET",
+  "port": 3100,
+  "path": "/get?id=geoname:4221195"
+}
+```
+
+## Response
+```javascript
+Status: 200
+{
+  "x-powered-by": "mapzen",
+  "charset": "utf8",
+  "cache-control": "public,max-age=60",
+  "server": "Pelias/0.0.0",
+  "access-control-allow-origin": "*",
+  "access-control-allow-methods": "GET",
+  "access-control-allow-headers": "X-Requested-With,content-type",
+  "access-control-allow-credentials": "true",
+  "content-type": "application/json; charset=utf-8",
+  "content-length": "317",
+  "etag": "W/\"13d-bc388729\"",
+  "date": "Thu, 16 Oct 2014 21:02:52 GMT",
+  "connection": "close"
+}
+```
+```javascript
+{
+  "type": "FeatureCollection",
+  "features": [
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -82.9207,
+          34.36094
+        ]
+      },
+      "properties": {
+        "name": "Sanders Grove Cemetery",
+        "admin0": "United States",
+        "admin1": "Georgia",
+        "admin2": "Hart County",
+        "text": "Sanders Grove Cemetery, Hart County, United States"
+      }
+    }
+  ],
+  "date": 1413493372842
+}
+```
+
+## Tests
+
+### ✓ valid geojson
+```javascript
+json.type.should.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+```
+
+### ✓ 200 ok
+```javascript
+response.statusCode.should.equal 200
+```
+
+### ✓ valid response
+```javascript
+now = new Date().getTime()
+should.exist json
+should.not.exist json.error
+json.date.should.be.within now-5000, now+5000
+```
+

sanitiser/_id.js

@@ -0,0 +1,73 @@
+// validate inputs, convert types and apply defaults
+// id generally looks like 'geoname:4163334' (type:id)
+// so, both type and id are required fields.
+
+function sanitize( req ){
+  
+  req.clean   = req.clean || {};
+  var params  = req.query;
+  var indeces = require('../query/indeces');
+  var delim   = ':';
+
+  // ensure params is a valid object
+  if( Object.prototype.toString.call( params ) !== '[object Object]' ){
+    params = {};
+  }
+
+  var errormessage = function(fieldname, message) {
+    return {
+      'error': true,
+      'message': message || ('invalid param \''+ fieldname + '\': text length, must be >0')
+    }
+  };
+
+  if(('string' === typeof params.id && !params.id.length) || params.id === undefined){
+    return errormessage('id');
+  }
+
+  if( params && params.id && params.id.length ){
+    req.clean.ids = [];
+    params.ids = Array.isArray(params.id) ? params.id : [params.id];
+    
+    // de-dupe
+    params.ids = params.ids.filter(function(item, pos) {
+      return params.ids.indexOf(item) == pos;
+    });
+
+    for (var i=0; i<params.ids.length; i++) {
+      var thisparam = params.ids[i];
+    
+      // basic format/ presence of ':'
+      if(thisparam.indexOf(delim) === -1) {
+        return errormessage(null, 'invalid: must be of the format type:id for ex: \'geoname:4163334\'');
+      }
+
+      var param = thisparam.split(delim);
+      var type= param[0];
+      var id  = param[1];
+
+      // id text
+      if('string' !== typeof id || !id.length){
+        return errormessage(thisparam);
+      }
+      // type text
+      if('string' !== typeof type || !type.length){
+        return errormessage(thisparam);
+      }
+      // type text must be one of the indeces
+      if(indeces.indexOf(type) == -1){
+        return errormessage('type', type + ' is invalid. It must be one of these values - [' + indeces.join(", ") + ']');
+      }
+      req.clean.ids.push({
+        id: id,
+        type: type
+      });
+    }
+  }
+  
+  return { 'error': false };
+
+}
+
+// export function
+module.exports = sanitize;

sanitiser/doc.js

@@ -0,0 +1,23 @@
+
+var logger = require('../src/logger'),
+    _sanitize = require('../sanitiser/_sanitize'),
+    sanitizers = {
+      id: require('../sanitiser/_id')
+    };
+
+var sanitize = function(req, cb) { _sanitize(req, sanitizers, cb); }
+
+// export sanitize for testing
+module.exports.sanitize = sanitize;
+
+// middleware
+module.exports.middleware = function( req, res, next ){
+  sanitize( req, function( err, clean ){
+    if( err ){
+      res.status(400); // 400 Bad Request
+      return next(err);
+    }
+    req.clean = clean;
+    next();
+  });
+};

test/ciao/doc/msuccess.coffee

@@ -0,0 +1,16 @@
+
+#> valid doc query
+path: '/doc?id=geoname:4221195&id=geoname:4193595'
+
+#? 200 ok
+response.statusCode.should.equal 200
+
+#? valid response
+now = new Date().getTime()
+should.exist json
+should.not.exist json.error
+json.date.should.be.within now-5000, now+5000
+
+#? valid geojson
+json.type.should.equal 'FeatureCollection'
+json.features.should.be.instanceof Array

test/ciao/doc/success.coffee

@@ -0,0 +1,16 @@
+
+#> valid doc query
+path: '/doc?id=geoname:4221195'
+
+#? 200 ok
+response.statusCode.should.equal 200
+
+#? valid response
+now = new Date().getTime()
+should.exist json
+should.not.exist json.error
+json.date.should.be.within now-5000, now+5000
+
+#? valid geojson
+json.type.should.equal 'FeatureCollection'
+json.features.should.be.instanceof Array

test/unit/query/suggest.js

@@ -63,9 +63,8 @@ module.exports.tests.precision = function(test, common) {
     {zoom:null, precision:1},
     {zoom:undefined, precision:1}
   ];
-
+  test('valid precision', function(t) {
-  test_cases.forEach( function( test_case ){
+    test_cases.forEach( function( test_case ){
-    test('valid precision where zoom = ' + test_case.zoom, function(t) {
       var query = generate({
         input: 'test', size: 10,
         lat: 0, lon: 0, zoom:test_case.zoom,
@@ -87,9 +86,9 @@ module.exports.tests.precision = function(test, common) {
           }
         }
       };
-      t.deepEqual(query, expected, 'valid suggest query');
+      t.deepEqual(query, expected, 'valid suggest query for zoom = ' + test_case.zoom);
-      t.end();
     });
+    t.end();
   });
 };
 

test/unit/run.js

@@ -6,7 +6,8 @@ var tests = [
   require('./controller/index'),
   require('./controller/suggest'),
   require('./controller/search'),
-  require('./sanitiser/sanitise'),
+  require('./sanitiser/suggest'),
+  require('./sanitiser/doc'),
   require('./query/indeces'),
   require('./query/suggest'),
   require('./query/search'),

test/unit/sanitiser/doc.js

@@ -0,0 +1,159 @@
+
+var doc  = require('../../../sanitiser/doc'),
+    _sanitize = doc.sanitize,
+    middleware = doc.middleware,
+    indeces = require('../../../query/indeces'),
+    delimiter = ':',
+    defaultLengthError = function(input) { return 'invalid param \''+ input + '\': text length, must be >0' },
+    defaultFormatError = 'invalid: must be of the format type:id for ex: \'geoname:4163334\'',
+    defaultError = 'invalid param \'id\': text length, must be >0',
+    defaultMissingTypeError = function(input) { 
+      var type = input.split(delimiter)[0];
+      return type + ' is invalid. It must be one of these values - [' + indeces.join(", ") + ']'},
+    defaultClean = { ids: [ { id: '123', type: 'geoname' } ] },
+    sanitize = function(query, cb) { _sanitize({'query':query}, cb); },
+    inputs = {
+      valid: [ 'geoname:1', 'osmnode:2', 'admin0:53', 'osmway:44', 'geoname:5' ],
+      invalid: [ ':', '', '::', 'geoname:', ':234', 'gibberish:23' ]
+    };
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('sanitize interface', function(t) {
+    t.equal(typeof sanitize, 'function', 'sanitize is a function');
+    t.equal(sanitize.length, 2, 'sanitize interface');
+    t.end();
+  });
+  test('middleware interface', function(t) {
+    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(middleware.length, 3, 'sanitizee has a valid middleware');
+    t.end();
+  });
+};
+
+module.exports.tests.sanitize_id = function(test, common) {
+  test('invalid input', function(t) {
+    inputs.invalid.forEach( function( input ){
+      sanitize({ id: input }, function( err, clean ){
+        switch (err) {
+          case defaultError:
+            t.equal(err, defaultError, input + ' is invalid input'); break;
+          case defaultLengthError(input):
+            t.equal(err, defaultLengthError(input), input + ' is invalid (missing id/type)'); break;
+          case defaultFormatError:
+            t.equal(err, defaultFormatError, input + ' is invalid (invalid format)'); break;
+          case defaultMissingTypeError(input):
+            t.equal(err, defaultMissingTypeError(input), input + ' is an unknown type'); break;
+          default: break;
+        }
+        t.equal(clean, undefined, 'clean not set');
+      });
+    });
+    t.end();
+  });
+
+  test('valid input', function(t) {
+    inputs.valid.forEach( function( input ){
+      var input_parts = input.split(delimiter);
+      var expected = { ids: [ { id: input_parts[1], type: input_parts[0] } ] };
+      sanitize({ id: input }, function( err, clean ){
+        t.equal(err, undefined, 'no error (' + input + ')' );
+        t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
+      });
+    });
+    t.end();
+  });
+};
+
+
+module.exports.tests.sanitize_ids = function(test, common) {
+  test('invalid input', function(t) {
+    sanitize({ id: inputs.invalid }, function( err, clean ){
+      var input = inputs.invalid[0]; // since it breaks on the first invalid element
+      switch (err) {
+        case defaultError:
+          t.equal(err, defaultError, input + ' is invalid input'); break;
+        case defaultLengthError(input):
+          t.equal(err, defaultLengthError(input), input + ' is invalid (missing id/type)'); break;
+        case defaultFormatError:
+          t.equal(err, defaultFormatError, input + ' is invalid (invalid format)'); break;
+        case defaultMissingTypeError(input):
+          t.equal(err, defaultMissingTypeError(input), input + ' is an unknown type'); break;
+        default: break;
+      }
+      t.equal(clean, undefined, 'clean not set');
+    });
+    t.end();
+  });
+
+  test('valid input', function(t) {
+    var expected={};
+    expected.ids = [];
+    inputs.valid.forEach( function( input ){
+      var input_parts = input.split(delimiter);
+      expected.ids.push({ id: input_parts[1], type: input_parts[0] });
+    });
+    sanitize({ id: inputs.valid }, function( err, clean ){
+      t.equal(err, undefined, 'no error' );
+      t.deepEqual(clean, expected, 'clean set correctly');
+    });
+    t.end();
+  });
+};
+
+module.exports.tests.de_dupe = function(test, common) {
+  var expected = { ids: [ { id: '1', type: 'geoname' }, { id: '2', type: 'osmnode' } ] };
+  test('duplicate ids', function(t) {
+    sanitize( { id: ['geoname:1', 'osmnode:2', 'geoname:1'] }, function( err, clean ){
+      t.equal(err, undefined, 'no error' );
+      t.deepEqual(clean, expected, 'clean set correctly');
+      t.end();
+    });
+  });
+};
+
+module.exports.tests.invalid_params = function(test, common) {
+  test('invalid params', function(t) {
+    sanitize( undefined, function( err, clean ){
+      t.equal(err, defaultError, 'handle invalid params gracefully');
+      t.end();
+    });
+  });
+};
+
+module.exports.tests.middleware_failure = function(test, common) {
+  test('middleware failure', function(t) {
+    var res = { status: function( code ){
+      t.equal(code, 400, 'status set');
+    }};
+    var next = function( message ){
+      t.equal(message, defaultError);
+      t.end();
+    };
+    middleware( {}, res, next );
+  });
+};
+
+module.exports.tests.middleware_success = function(test, common) {
+  test('middleware success', function(t) {
+    var req = { query: { id: 'geoname' + delimiter + '123' }};
+    var next = function( message ){
+      t.equal(message, undefined, 'no error message set');
+      t.deepEqual(req.clean, defaultClean);
+      t.end();
+    };
+    middleware( req, undefined, next );
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SANTIZE /doc ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/suggest.js

@@ -26,25 +26,25 @@ module.exports.tests.sanitize_input = function(test, common) {
     invalid: [ '', 100, null, undefined, new Date() ],
     valid: [ 'a', 'aa', 'aaaaaaaa' ]
   };
-  inputs.invalid.forEach( function( input ){
+  test('invalid input', function(t) {  
-    test('invalid input', function(t) {
+    inputs.invalid.forEach( function( input ){
       sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){
-        t.equal(err, 'invalid param \'input\': text length, must be >0', 'invalid input');
+        t.equal(err, 'invalid param \'input\': text length, must be >0', input + ' is an invalid input');
         t.equal(clean, undefined, 'clean not set');
-        t.end();
       });
     });
+    t.end();
   });
-  inputs.valid.forEach( function( input ){
+  test('valid input', function(t) {  
-    test('valid input', function(t) {
+    inputs.valid.forEach( function( input ){
       sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.input = input;
         t.equal(err, undefined, 'no error');
-        t.deepEqual(clean, expected, 'clean set correctly');
+        t.deepEqual(clean, expected, 'clean set correctly (' + input + ')');
-        t.end();
       });
     });
+    t.end();
   });
 };
 
@@ -53,25 +53,25 @@ module.exports.tests.sanitize_lat = function(test, common) {
     invalid: [ -181, -120, -91, 91, 120, 181 ],
     valid: [ 0, 45, 90, -0, '0', '45', '90' ]
   };
-  lats.invalid.forEach( function( lat ){
+  test('invalid lat', function(t) {  
-    test('invalid lat', function(t) {
+    lats.invalid.forEach( function( lat ){
       sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){
-        t.equal(err, 'invalid param \'lat\': must be >-90 and <90', 'invalid latitude');
+        t.equal(err, 'invalid param \'lat\': must be >-90 and <90', lat + ' is an invalid latitude');
         t.equal(clean, undefined, 'clean not set');
-        t.end();
       });
     });
+    t.end();
   });
-  lats.valid.forEach( function( lat ){
+  test('valid lat', function(t) {  
-    test('valid lat', function(t) {
+    lats.valid.forEach( function( lat ){
       sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.lat = parseFloat( lat );
         t.equal(err, undefined, 'no error');
-        t.deepEqual(clean, expected, 'clean set correctly');
+        t.deepEqual(clean, expected, 'clean set correctly (' + lat + ')');
-        t.end();
       });
     });
+    t.end();
   });
 };
 
@@ -80,25 +80,26 @@ module.exports.tests.sanitize_lon = function(test, common) {
     invalid: [ -360, -181, 181, 360 ],
     valid: [ -180, -1, -0, 0, 45, 90, '-180', '0', '180' ]
   };
-  lons.invalid.forEach( function( lon ){
+  test('invalid lon', function(t) {  
-    test('invalid lon', function(t) {
+    lons.invalid.forEach( function( lon ){
       sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){
-        t.equal(err, 'invalid param \'lon\': must be >-180 and <180', 'invalid longitude');
+        t.equal(err, 'invalid param \'lon\': must be >-180 and <180', lon + ' is an invalid longitude');
         t.equal(clean, undefined, 'clean not set');
-        t.end();
+        
       });
     });
+    t.end();
   });
-  lons.valid.forEach( function( lon ){
+  test('valid lon', function(t) {  
-    test('valid lon', function(t) {
+    lons.valid.forEach( function( lon ){
       sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){
         var expected = JSON.parse(JSON.stringify( defaultClean ));
         expected.lon = parseFloat( lon );
         t.equal(err, undefined, 'no error');
-        t.deepEqual(clean, expected, 'clean set correctly');
+        t.deepEqual(clean, expected, 'clean set correctly (' + lon + ')');
-        t.end();
       });
     });
+    t.end();
   });
 };
 
@@ -241,7 +242,7 @@ module.exports.tests.middleware_success = function(test, common) {
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('SANTIZE /sanitise ' + name, testFunction);
+    return tape('SANTIZE /suggest ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){