Merge branch 'master' into address-details

.gitignore

@@ -1,2 +1,5 @@
 node_modules
-*.log
+coverage
+.idea
+*.log
+reports

.jshintignore

@@ -1 +1,3 @@
 node_modules
+coverage
+reports

DOCS.md

@@ -0,0 +1,114 @@
+## /search
+
+Full text search endpoint which queries the elasticsearch doc store, slightly slower than suggest.
+
+#### Required Parameters
+* **input**: query string
+
+#### Optional Parameters
+* **lat**: latitude
+* **lon**: longitude
+* **zoom**: zoom level from which you wish to view the world
+* **size**: number of results requested (defaults to 10)
+* **layers**: datasets you wish to query (defaults to ```poi,admin,address```). 
+  * valid values are ```poi```, ```admin``` or ```address``` 
+    * ```poi``` expands internally to ```geoname```, ```osmnode```, ```osmway``` 
+    * ```admin``` expands to ```admin0```, ```admin1```, ```admin2```, ```neighborhood```, ```locality```, ```local_admin```
+    * ```address``` expands to ```osmaddress```, ```openaddresses```
+  * can also be specific to one particular dataset, for example ```geoname```
+* **bbox**: the bounding box from which you want all your results to come
+  * can be one of the following comma separated string values
+    * bottom left lat, bottom left lon, top right lat, top right lon
+    * left, bottom, right, top
+    * min longitude, min latitude, max longitude, max latitude 
+
+
+## /search/coarse
+
+This is a coarse forward geocoder endpoint which only searches admin dataset layers.
+
+#### Required Parameters
+* **input**: query string
+
+#### Optional Parameters
+* **lat**: latitude
+* **lon**: longitude
+* **zoom**: zoom level from which you wish to view the world
+* **bbox**: the bounding box frome which you want all your results to come
+* **size**: (defaults to 10)
+* **layers**: (defaults to ```admin```)
+
+
+## /suggest
+
+The autocomplete endpoint, it offers fast response time. Mixes results from around the provided lat/lon and also from precision level 1 and 3.
+
+#### Required Parameters
+* **input**: query string
+* **lat**: latitude
+* **lon**: longitude
+  * lat/lon are **required** currently because of this [open issue](https://github.com/elasticsearch/elasticsearch/issues/6444)
+
+#### Optional Parameters
+* **zoom**: zoom level from which you wish to view the world
+* **size**: number of results requested (defaults to 10)
+* **layers**: datasets you wish to query (defaults to ```poi,admin,address```)
+
+
+## /suggest/coarse
+
+Only queries the admin layers.
+
+#### Required Parameters
+* **input**: query string
+* **lat**: latitude from where you are searching
+* **lon**: longitude
+  * lat/lon are **required** currently because of this [open issue](https://github.com/elasticsearch/elasticsearch/issues/6444)
+
+#### Optional Parameters
+* **zoom**: zoom level from which you wish to view the world
+* **size**: number of results requested (defaults to 10)
+* **layers**: datasets you wish to query (defaults to ```admin```)
+
+
+## /suggest/nearby
+
+Works as autocomplete for places located near a latitude/longitude, this endpoint is the same as ```/suggest``` but the results are all from within 50 kilometers of the specified point.  Unlike ```/suggest```, ```/suggest/nearby``` does not mix results from different precision levels (500km, 1000km etc from lat/lon).
+
+#### Required Parameters
+* **input**: query string
+* **lat**: latitude
+* **lon**: longitude
+  * lat/lon are **required** currently because of this [open issue](https://github.com/elasticsearch/elasticsearch/issues/6444)
+
+#### Optional Parameters
+* **zoom**: zoom level from which you wish to view the world
+* **size**: number of results you need (defaults to 10)
+* **layers**: datasets you wish to query (defaults to ```poi,admin,address```)
+
+
+## /reverse
+
+Reverse geocoding endpoint.
+
+#### Required Parameters
+* **lat**: latitude
+* **lon**: longitude
+
+#### Optional Parameters
+* **zoom**: zoom level from which you wish to view the world
+* **bbox**: bounding box
+* **layers**: (defaults to ```poi,admin,address```)
+
+
+## /doc
+
+Retrieves a document or multiple documents at once.
+
+#### Required Parameters
+* one of **id** or **ids**
+  * **id**:
+    * unique id of the document to be retrieved
+    * should be in the form of type:id, for example: ```geoname:4163334```
+  * **ids**:
+    * if multiple docs are to be fetched in bulk, an array of ids

README.md

@@ -58,6 +58,12 @@ $ npm test
 $ npm run docs
 ```
 
+### Code Coverage
+
+```bash
+$ npm run coverage
+```
+
 ### Continuous Integration
 
 Travis tests every release against node version `0.10`

app.js

@@ -38,7 +38,7 @@ app.get( '/suggest/nearby',
 		sanitisers.suggest.middleware, 
 		controllers.suggest(undefined, undefined, require('./helper/queryMixer').suggest_nearby) );
 app.get( '/suggest/coarse', 
-		sanitisers.suggest.middleware, 
+		sanitisers.coarse.middleware, 
 		controllers.suggest(undefined, undefined, require('./helper/queryMixer').coarse) );
 
 // search API

controller/index.js

@@ -1,22 +1,31 @@
 
 var pkg = require('../package');
+var markdown = require('markdown').markdown;
+var fs = require('fs');
 
 function setup(){
 
-  function controller( req, res, next ){
+  var styleString = '<style>html{font-family:monospace}</style>';
-
+  var text = '# Pelias API\n';
-    // stats
+  text += '### Version: ['+ pkg.version+ '](https://github.com/pelias/api/releases)\n';
+  text += fs.readFileSync( './DOCS.md', 'utf8');
+  var indexHtml = styleString + markdown.toHTML(text);
+
+  function controller( req, res, next ) {
+    if (req.accepts('html')) {
+      res.send(indexHtml);
+      return;
+    }
+    // default behaviour
     res.json({
       name: pkg.name,
       version: {
         number: pkg.version
       }
     });
-
   }
 
   return controller;
-
 }
 
-module.exports = setup;
+module.exports = setup;

middleware/500.js

@@ -1,9 +1,12 @@
+var logger = require( '../src/logger' );
 
 // handle application errors
 function middleware(err, req, res, next) {
+  logger.error( 'Error:', err );
+  logger.error( 'Stack trace:', err.trace );
   res.header('Cache-Control','no-cache');
   if( res.statusCode < 400 ){ res.status(500); }
-  res.json({ error: err });
+  res.json({ error: 'internal server error' });
 }
 
-module.exports = middleware;
+module.exports = middleware;

package.json

@@ -1,16 +1,17 @@
 {
   "name": "pelias-api",
   "author": "mapzen",
-  "version": "0.0.0",
+  "version": "1.1.7",
   "description": "Pelias API",
   "homepage": "https://github.com/pelias/api",
   "license": "MIT",
   "main": "index.js",
   "scripts": {
     "start": "node index.js",
-    "test": "npm run unit && npm run ciao",
+    "test": "npm run unit && npm run coverage",
     "unit": "node test/unit/run.js | tap-spec",
     "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao",
+    "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
     "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
     "docs": "rm -r docs; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs"
   },
@@ -37,16 +38,19 @@
     "geojson": "^0.2.0",
     "geojson-extent": "^0.3.1",
     "geopipes-elasticsearch-backend": "0.0.12",
-    "pelias-suggester-pipeline": "2.0.2",
     "is-object": "^1.0.1",
-    "pelias-esclient": "0.0.25"
+    "markdown": "0.5.0",
+    "pelias-esclient": "0.0.25",
+    "pelias-suggester-pipeline": "2.0.2"
   },
   "devDependencies": {
     "ciao": "^0.3.4",
+    "istanbul": "^0.3.13",
     "jshint": "^2.5.6",
+    "nsp": "^0.3.0",
     "precommit-hook": "^1.0.7",
-    "tape": "^2.13.4",
+    "proxyquire": "^1.4.0",
     "tap-spec": "^0.2.0",
-    "nsp": "^0.3.0"
+    "tape": "^2.13.4"
   }
 }

query/reverse.js

@@ -1,7 +1,6 @@
 
-var logger = require('../src/logger'),
+var queries = require('geopipes-elasticsearch-backend').queries,
-    queries = require('geopipes-elasticsearch-backend').queries,
+    sort = require('./sort');
-    sort = require('../query/sort');
 
 function generate( params ){
 
@@ -13,7 +12,17 @@ function generate( params ){
   var query  =  queries.distance( centroid, { size: params.size || 1 } );
   query.sort = query.sort.concat(sort);
 
+  if ( params.categories && params.categories.length > 0 ) {
+    addCategoriesFilter( query, params.categories );
+  }
+
   return query;
 }
 
+function addCategoriesFilter( query, categories ) {
+  query.query.filtered.filter.bool.must.push({
+    terms: { category: categories }
+  });
+}
+
 module.exports = generate;

sanitiser/_categories.js

@@ -0,0 +1,38 @@
+
+var isObject = require('is-object');
+
+// validate inputs, convert types and apply defaults
+function sanitize( req ){
+
+  var clean = req.clean || {};
+  var params= req.query;
+
+  // ensure the input params are a valid object
+  if( !isObject( params ) ){
+    params = {};
+  }
+
+  // default case (no categories specified in GET params)
+  if('string' !== typeof params.categories || !params.categories.length){
+    clean.categories = [];
+  }
+  else {
+    // parse GET params
+    clean.categories = params.categories.split(',')
+      .map(function (cat) {
+        return cat.toLowerCase().trim(); // lowercase inputs
+      })
+      .filter( function( cat ) {
+        return ( cat.length > 0 );
+      });
+  }
+
+  // pass validated params to next middleware
+  req.clean = clean;
+
+  return { 'error': false };
+
+}
+
+// export function
+module.exports = sanitize;

sanitiser/_layers.js

@@ -33,7 +33,7 @@ function sanitize( req ){
     if( -1 === alias_indeces.indexOf( layers[x] ) ){
       return {
         'error': true,
-        'message': 'invalid param \'layer\': must be one or more of ' + alias_indeces.join(',')
+        'message': 'invalid param \'layers\': must be one or more of ' + alias_indeces.join(',')
       };
     }
   }

sanitiser/reverse.js

@@ -1,6 +1,5 @@
 
-var logger = require('../src/logger'),
+var _sanitize = require('../sanitiser/_sanitize'),
-    _sanitize = require('../sanitiser/_sanitize'),
     sanitiser = {
       latlonzoom: function( req ) {
         var geo = require('../sanitiser/_geo');
@@ -11,6 +10,10 @@ var logger = require('../src/logger'),
       size: function( req ) {
         var size = require('../sanitiser/_size');
         return size(req, 1);
+      },
+      categories: function ( req ) {
+        var categories = require('../sanitiser/_categories');
+        return categories(req);
       }
     };
 

test/ciao/index.coffee

@@ -6,7 +6,7 @@ path: '/'
 response.statusCode.should.equal 200
 
 #? content-type header correctly set
-response.should.have.header 'Content-Type','application/json; charset=utf-8'
+response.should.have.header 'Content-Type','text/html; charset=utf-8'
 
 #? charset header correctly set
 response.should.have.header 'Charset','utf8'
@@ -20,8 +20,3 @@ response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/
 
 #? vanity header correctly set
 response.should.have.header 'X-Powered-By','mapzen'
-
-#? should respond in json with server info
-should.exist json
-should.exist json.name
-should.exist json.version

test/ciao/jsonp.coffee

@@ -1,10 +0,0 @@
-
-#> jsonp
-path: '/?callback=test'
-
-#? content-type header correctly set
-response.should.have.header 'Content-Type','application/javascript; charset=utf-8'
-
-#? should respond with jsonp
-should.exist response.body
-response.body.substr(0,5).should.equal 'test(';

test/unit/controller/index.js

@@ -11,9 +11,15 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-module.exports.tests.info = function(test, common) {
+module.exports.tests.info_json = function(test, common) {
-  test('returns server info', function(t) {
+  test('returns server info in json', function(t) {
     var controller = setup();
+    var req = {
+      accepts: function (format) {
+        t.equal(format, 'html', 'check for Accepts:html');
+        return false;
+      }
+    };
     var res = { json: function( json ){
       t.equal(typeof json, 'object', 'returns json');
       t.equal(typeof json.name, 'string', 'name');
@@ -21,7 +27,39 @@ module.exports.tests.info = function(test, common) {
       t.equal(typeof json.version.number, 'string', 'version number');
       t.end();
     }};
-    controller( null, res );
+    controller( req, res );
+  });
+};
+
+module.exports.tests.info_html = function(test, common) {
+  test('returns server info in html', function(t) {
+
+    var style = '<style>html{font-family:monospace}</style>';
+    var mockText = 'this text should show up in the html content';
+    var fsMock = {
+      readFileSync: function (path, format) {
+        t.equal(path, './DOCS.md', 'open DOCS.md file');
+        t.equal(format, 'utf8', 'file format');
+        return mockText;
+      }
+    };
+
+    var proxyquire = require('proxyquire');
+    var setup = proxyquire('../../../controller/index', { 'fs': fsMock });
+
+    var controller = setup();
+    var req = {
+      accepts: function () {
+        return true;
+      }
+    };
+    var res = { send: function( content ){
+      t.equal(typeof content, 'string', 'returns string');
+      t.assert(content.indexOf(style) === 0, 'style set');
+      t.assert(content.indexOf(mockText) !== -1, 'file content added');
+      t.end();
+    }};
+    controller( req, res );
   });
 };
 

test/unit/query/reverse.js

@@ -111,6 +111,71 @@ module.exports.tests.query = function(test, common) {
     });
     t.end();
   });
+
+  test('valid query with categories', function(t) {
+    var params = { lat: 29.49136, lon: -82.50622, categories: ['food', 'education', 'entertainment'] };
+    var query = generate(params);
+
+    var expected = {
+      'query': {
+        'filtered': {
+          'query': {
+            'match_all': {}
+          },
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'geo_distance': {
+                    'distance': '50km',
+                    'distance_type': 'plane',
+                    'optimize_bbox': 'indexed',
+                    '_cache': true,
+                    'center_point': {
+                      'lat': '29.49',
+                      'lon': '-82.51'
+                    }
+                  }
+                },
+                {
+                  'terms': {
+                    'category': params.categories
+                  }
+                }
+              ]
+            }
+          }
+        }
+      },
+      'sort': [
+        '_score',
+        {
+          '_geo_distance': {
+            'center_point': {
+              'lat': 29.49136,
+              'lon': -82.50622
+            },
+            'order': 'asc',
+            'unit': 'km'
+          }
+        }
+      ].concat(sort.slice(1)),
+      'size': 1,
+      'track_scores': true
+    };
+
+    t.deepEqual(query, expected, 'valid reverse query with categories');
+
+    // test different sizes
+    var sizes = [1,2,10,undefined,null];
+    sizes.forEach( function(size) {
+      params.size = size;
+      query = generate(params);
+      expected.size = size ? size : 1;
+      t.deepEqual(query, expected, 'valid reverse query for size: '+ size);
+    });
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {

test/unit/sanitiser/reverse.js

@@ -9,7 +9,8 @@ var suggest  = require('../../../sanitiser/reverse'),
                                 'locality', 'local_admin', 'osmaddress', 'openaddresses' ], 
                       lon: 0,
                       size: 1,
-                      details: true
+                      details: true,
+                      categories: []
                     },
     sanitize = function(query, cb) { _sanitize({'query':query}, cb); };
 
@@ -173,7 +174,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
   });
   test('invalid layer', function(t) {
     sanitize({ layers: 'test_layer', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      var msg = 'invalid param \'layer\': must be one or more of ';
+      var msg = 'invalid param \'layers\': must be one or more of ';
       t.true(err.match(msg), 'invalid layer requested');
       t.true(err.length > msg.length, 'invalid error message');
       t.end();
@@ -240,6 +241,45 @@ module.exports.tests.sanitize_layers = function(test, common) {
   });
 };
 
+module.exports.tests.sanitize_categories = function(test, common) {
+  var queryParams = { input: 'test', lat: 0, lon: 0 };
+  test('unspecified', function(t) {
+    queryParams.categories = undefined;
+    sanitize(queryParams, function( err, clean ){
+      t.deepEqual(clean.categories, defaultClean.categories, 'default to empty categories array');
+      t.end();
+    });
+  });
+  test('single category', function(t) {
+    queryParams.categories = 'food';
+    sanitize(queryParams, function( err, clean ){
+      t.deepEqual(clean.categories, ['food'], 'category set');
+      t.end();
+    });
+  });
+  test('multiple categories', function(t) {
+    queryParams.categories = 'food,education,nightlife';
+    sanitize(queryParams, function( err, clean ){
+      t.deepEqual(clean.categories, ['food', 'education', 'nightlife'], 'categories set');
+      t.end();
+    });
+  });
+  test('whitespace and empty strings', function(t) {
+    queryParams.categories = 'food, , nightlife ,';
+    sanitize(queryParams, function( err, clean ){
+      t.deepEqual(clean.categories, ['food', 'nightlife'], 'categories set');
+      t.end();
+    });
+  });
+  test('all empty strings', function(t) {
+    queryParams.categories = ', ,  ,';
+    sanitize(queryParams, function( err, clean ){
+      t.deepEqual(clean.categories, defaultClean.categories, 'empty strings filtered out');
+      t.end();
+    });
+  });
+};
+
 module.exports.tests.middleware_failure = function(test, common) {
   test('middleware failure', function(t) {
     var res = { status: function( code ){

test/unit/sanitiser/search.js

@@ -315,7 +315,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
   });
   test('invalid layer', function(t) {
     sanitize({ layers: 'test_layer', input: 'test' }, function( err, clean ){
-      var msg = 'invalid param \'layer\': must be one or more of ';
+      var msg = 'invalid param \'layers\': must be one or more of ';
       t.true(err.match(msg), 'invalid layer requested');
       t.true(err.length > msg.length, 'invalid error message');
       t.end();

test/unit/sanitiser/suggest.js

@@ -284,7 +284,7 @@ module.exports.tests.sanitize_layers = function(test, common) {
   });
   test('invalid layer', function(t) {
     sanitize({ layers: 'test_layer', input: 'test', lat: 0, lon: 0 }, function( err, clean ){
-      var msg = 'invalid param \'layer\': must be one or more of ';
+      var msg = 'invalid param \'layers\': must be one or more of ';
       t.true(err.match(msg), 'invalid layer requested');
       t.true(err.length > msg.length, 'invalid error message');
       t.end();