diff --git a/.travis.yml b/.travis.yml index 607bac83..82ec939c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: node_js script: "npm run unit" node_js: - - "0.11" - "0.10" \ No newline at end of file diff --git a/app.js b/app.js new file mode 100644 index 00000000..e3892851 --- /dev/null +++ b/app.js @@ -0,0 +1,39 @@ + +var app = require('express')(); + +/** ----------------------- middleware ----------------------- **/ + +app.use( require('./middleware/toobusy') ); // should be first +app.use( require('./middleware/headers') ); +app.use( require('./middleware/cors') ); +app.use( require('./middleware/jsonp') ); + +/** ----------------------- sanitisers ----------------------- **/ + +var sanitisers = {}; +sanitisers.sanitiser = require('./sanitiser/sanitise'); + +/** ----------------------- controllers ----------------------- **/ + +var controllers = {}; +controllers.index = require('./controller/index'); +controllers.suggest = require('./controller/suggest'); +controllers.search = require('./controller/search'); + +/** ----------------------- routes ----------------------- **/ + +// api root +app.get( '/', controllers.index() ); + +// suggest API +app.get( '/suggest', sanitisers.sanitiser.middleware, controllers.suggest() ); + +// search API +app.get( '/search', sanitisers.sanitiser.middleware, controllers.search() ); + +/** ----------------------- error middleware ----------------------- **/ + +app.use( require('./middleware/404') ); +app.use( require('./middleware/500') ); + +module.exports = app; \ No newline at end of file diff --git a/controller/index.js b/controller/index.js index 6936a982..10f2d9a4 100644 --- a/controller/index.js +++ b/controller/index.js @@ -1,16 +1,22 @@ var pkg = require('../package'); -function controller( req, res, next ){ +function setup(){ - // stats - res.json({ - name: pkg.name, - version: { - number: pkg.version - } - }); + function controller( req, res, next ){ + + // stats + res.json({ + name: pkg.name, + version: { + number: pkg.version + } + }); + + } + + return controller; } -module.exports = controller; \ No newline at end of file +module.exports = setup; \ No newline at end of file diff --git a/controller/search.js b/controller/search.js index dd021120..e396c78e 100644 --- a/controller/search.js +++ b/controller/search.js @@ -1,36 +1,42 @@ -var query = require('../query/search'), - backend = require('../src/backend'); +function setup( backend, query ){ -function controller( req, res, next ){ + // allow overriding of dependencies + backend = backend || require('../src/backend'); + query = query || require('../query/search'); - // backend command - var cmd = { - index: 'pelias', - body: query( req.clean ) - }; + function controller( req, res, next ){ - // query backend - backend().client.search( cmd, function( err, data ){ + // backend command + var cmd = { + index: 'pelias', + body: query( req.clean ) + }; - var docs = []; + // query backend + backend().client.search( cmd, function( err, data ){ - // handle backend errors - if( err ){ return next( err ); } + var docs = []; - if( data && data.hits && data.hits.total){ - docs = data.hits.hits.map( function( hit ){ - return hit._source; - }); - } + // handle backend errors + if( err ){ return next( err ); } + + if( data && data.hits && data.hits.total){ + docs = data.hits.hits.map( function( hit ){ + return hit._source; + }); + } - // respond - return res.status(200).json({ - date: new Date().getTime(), - body: docs + // respond + return res.status(200).json({ + date: new Date().getTime(), + body: docs + }); }); - }); + } + + return controller; } -module.exports = controller; \ No newline at end of file +module.exports = setup; \ No newline at end of file diff --git a/controller/suggest.js b/controller/suggest.js index c6967186..b62b2fe6 100644 --- a/controller/suggest.js +++ b/controller/suggest.js @@ -1,35 +1,41 @@ -var query = require('../query/suggest'), - backend = require('../src/backend'); +function setup( backend, query ){ -function controller( req, res, next ){ + // allow overriding of dependencies + backend = backend || require('../src/backend'); + query = query || require('../query/suggest'); - // backend command - var cmd = { - index: 'pelias', - body: query( req.clean ) - }; + function controller( req, res, next ){ - // query backend - backend().client.suggest( cmd, function( err, data ){ + // backend command + var cmd = { + index: 'pelias', + body: query( req.clean ) + }; - var docs = []; + // query backend + backend().client.suggest( cmd, function( err, data ){ - // handle backend errors - if( err ){ return next( err ); } + var docs = []; - // map response to a valid FeatureCollection - if( data && Array.isArray( data.pelias ) && data.pelias.length ){ - docs = data['pelias'][0].options || []; - } + // handle backend errors + if( err ){ return next( err ); } - // respond - return res.status(200).json({ - date: new Date().getTime(), - body: docs + // map response to a valid FeatureCollection + if( data && Array.isArray( data.pelias ) && data.pelias.length ){ + docs = data['pelias'][0].options || []; + } + + // respond + return res.status(200).json({ + date: new Date().getTime(), + body: docs + }); }); - }); + } + + return controller; } -module.exports = controller; \ No newline at end of file +module.exports = setup; \ No newline at end of file diff --git a/docs/404.md b/docs/404.md index e683e480..df08c981 100644 --- a/docs/404.md +++ b/docs/404.md @@ -1,6 +1,6 @@ # invalid path -*Generated: Fri Sep 12 2014 20:51:44 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 404 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=300", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=300", "content-type": "application/json; charset=utf-8", "content-length": "35", "etag": "W/\"23-dfdfa185\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` @@ -39,19 +39,14 @@ Status: 404 ## Tests -### ✓ cache-control header correctly set -```javascript -response.should.have.header 'Cache-Control','public,max-age=300' -``` - -### ✓ content-type header correctly set +### ✓ not found ```javascript -response.should.have.header 'Content-Type','application/json; charset=utf-8' +response.statusCode.should.equal 404 ``` -### ✓ not found +### ✓ cache-control header correctly set ```javascript -response.statusCode.should.equal 404 +response.should.have.header 'Cache-Control','public,max-age=300' ``` ### ✓ should respond in json with server info @@ -61,3 +56,8 @@ should.exist json.error json.error.should.equal 'not found: invalid path' ``` +### ✓ content-type header correctly set +```javascript +response.should.have.header 'Content-Type','application/json; charset=utf-8' +``` + diff --git a/docs/cors.md b/docs/cors.md index 0df8dc00..e30c37b7 100644 --- a/docs/cors.md +++ b/docs/cors.md @@ -1,6 +1,6 @@ # cross-origin resource sharing -*Generated: Fri Sep 12 2014 20:51:44 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "50", "etag": "W/\"32-85536434\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` diff --git a/docs/index.md b/docs/index.md index 963ab240..b3eee35e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # api root -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "50", "etag": "W/\"32-85536434\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` @@ -42,6 +42,12 @@ Status: 200 ## Tests +### ✓ server header correctly set +```javascript +response.should.have.header 'Server' +response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/ +``` + ### ✓ content-type header correctly set ```javascript response.should.have.header 'Content-Type','application/json; charset=utf-8' @@ -57,20 +63,14 @@ response.statusCode.should.equal 200 response.should.have.header 'Cache-Control','public,max-age=60' ``` -### ✓ charset header correctly set -```javascript -response.should.have.header 'Charset','utf8' -``` - -### ✓ server header correctly set +### ✓ vanity header correctly set ```javascript -response.should.have.header 'Server' -response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/ +response.should.have.header 'X-Powered-By','mapzen' ``` -### ✓ vanity header correctly set +### ✓ charset header correctly set ```javascript -response.should.have.header 'X-Powered-By','mapzen' +response.should.have.header 'Charset','utf8' ``` ### ✓ should respond in json with server info diff --git a/docs/jsonp.md b/docs/jsonp.md index a67e5c40..382f3104 100644 --- a/docs/jsonp.md +++ b/docs/jsonp.md @@ -1,6 +1,6 @@ # jsonp -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/javascript; charset=utf-8", "content-length": "57", "etag": "W/\"39-b8a2aba1\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` diff --git a/docs/suggest/success.md b/docs/suggest/success.md index abcc5db4..943b2bfd 100644 --- a/docs/suggest/success.md +++ b/docs/suggest/success.md @@ -1,6 +1,6 @@ # valid suggest query -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,22 +18,22 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "1248", - "etag": "W/\"o9NALcf9i0O3JoLO7pfqog==\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "etag": "W/\"H7TGcVqWuu8sAb6aBhpd6A==\"", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` ```javascript { - "date": 1410551504928, + "date": 1411048419796, "body": [ { "text": "ACRELÂNDIA, Brazil", @@ -121,6 +121,11 @@ Status: 200 ## Tests +### ✓ 200 ok +```javascript +response.statusCode.should.equal 200 +``` + ### ✓ valid response ```javascript now = new Date().getTime() @@ -132,8 +137,3 @@ should.exist json.body json.body.should.be.instanceof Array ``` -### ✓ 200 ok -```javascript -response.statusCode.should.equal 200 -``` - diff --git a/index.js b/index.js index 39207611..fe6c73b9 100644 --- a/index.js +++ b/index.js @@ -1,26 +1,17 @@ -var app = require('express')(); - -/** ----------------------- middleware ----------------------- **/ - -app.use( require('./middleware/headers') ); -app.use( require('./middleware/cors') ); -app.use( require('./middleware/jsonp') ); - -/** ----------------------- routes ----------------------- **/ - -// api root -app.get( '/', require('./controller/index') ); - -// suggest API -app.get( '/suggest', require('./sanitiser/sanitise'), require('./controller/suggest') ); - -// search API -app.get( '/search', require('./sanitiser/sanitise'), require('./controller/search') ); - -/** ----------------------- error middleware ----------------------- **/ - -app.use( require('./middleware/404') ); -app.use( require('./middleware/500') ); - -app.listen( process.env.PORT || 3100 ); \ No newline at end of file +var cluster = require('cluster'), + app = require('./app'), + multicore = false, + port = ( process.env.PORT || 3100 ); + +/** cluster webserver across all cores **/ +if( multicore ){ + // @todo: not finished yet + // cluster(app) + // .use(cluster.stats()) + // .listen( process.env.PORT || 3100 ); +} +else { + console.log( 'listening on ' + port ); + app.listen( process.env.PORT || 3100 ); +} diff --git a/middleware/toobusy.js b/middleware/toobusy.js new file mode 100644 index 00000000..bbe6784f --- /dev/null +++ b/middleware/toobusy.js @@ -0,0 +1,19 @@ + +// middleware which blocks requests when the eventloop is too busy +var toobusy = require('toobusy'); + +function middleware(req, res, next){ + if( toobusy() ){ + res.status(503); // Service Unavailable + return next('Server Overwhelmed'); + } + return next(); +} + +// calling .shutdown allows your process to exit normally +process.on('SIGINT', function() { + toobusy.shutdown(); + process.exit(); +}); + +module.exports = middleware; \ No newline at end of file diff --git a/package.json b/package.json index 46c15a5d..76962055 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "test": "npm run unit && npm run ciao", "unit": "node test/unit/run.js | tap-spec", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", + "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;", "docs": "rm -r docs; cd test/ciao; node ../../node_modules/ciao/bin/ciao -c ../ciao.json . -d ../../docs" }, "repository": { @@ -33,11 +34,13 @@ "dependencies": { "express": "^4.8.8", "geopipes-elasticsearch-backend": "0.0.7", - "pelias-esclient": "0.0.25" + "pelias-esclient": "0.0.25", + "toobusy": "^0.2.4" }, "devDependencies": { "ciao": "^0.3.4", "tape": "^2.13.4", - "tap-spec": "^0.2.0" + "tap-spec": "^0.2.0", + "nsp": "^0.3.0" } } diff --git a/query/search.js b/query/search.js index e17d0838..a504acf4 100644 --- a/query/search.js +++ b/query/search.js @@ -22,7 +22,7 @@ function generate( params ){ } } }, - "size": 30 + "size": params.size }; logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); diff --git a/query/suggest.js b/query/suggest.js index b54e2518..9ba2e5c8 100644 --- a/query/suggest.js +++ b/query/suggest.js @@ -3,7 +3,7 @@ var logger = require('../src/logger'); // Build pelias suggest query function generate( params ){ - + var cmd = { 'pelias' : { 'text' : params.input, @@ -23,7 +23,7 @@ function generate( params ){ } }; - logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); + // logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); return cmd; } diff --git a/sanitiser/sanitise.js b/sanitiser/sanitise.js index d7dfa60a..85f972f6 100644 --- a/sanitiser/sanitise.js +++ b/sanitiser/sanitise.js @@ -21,7 +21,7 @@ function sanitize( params, cb ){ // total results var size = parseInt( params.size, 10 ); if( !isNaN( size ) ){ - clean.size = Math.min( size, 40 ); // max + clean.size = Math.min( Math.max( size, 1 ), 40 ); // max } else { clean.size = 10; // default } @@ -33,7 +33,7 @@ function sanitize( params, cb ){ }); for( var x=0; x0', + defaultClean = { input: 'test', lat: 0, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood' ], lon: 0, size: 10, zoom: 10 }; + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('sanitize interface', function(t) { + t.equal(typeof sanitize, 'function', 'sanitize is a function'); + t.equal(sanitize.length, 2, 'sanitize interface'); + t.end(); + }); + test('middleware interface', function(t) { + t.equal(typeof sanitize.middleware, 'function', 'middleware is a function'); + t.equal(sanitize.middleware.length, 3, 'sanitize is valid middleware'); + t.end(); + }); +}; + +module.exports.tests.sanitize_input = function(test, common) { + var inputs = { + invalid: [ '', 100, null, undefined, new Date() ], + valid: [ 'a', 'aa', 'aaaaaaaa' ] + }; + inputs.invalid.forEach( function( input ){ + test('invalid input', function(t) { + sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){ + t.equal(err, 'invalid param \'input\': text length, must be >0', 'invalid input'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + inputs.valid.forEach( function( input ){ + test('valid input', function(t) { + sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.input = input; + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_lat = function(test, common) { + var lats = { + invalid: [ -1, -45, -90, 91, 120, 181 ], + valid: [ 0, 45, 90, -0, '0', '45', '90' ] + }; + lats.invalid.forEach( function( lat ){ + test('invalid lat', function(t) { + sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){ + t.equal(err, 'invalid param \'lat\': must be >0 and <90', 'invalid latitude'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + lats.valid.forEach( function( lat ){ + test('valid lat', function(t) { + sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.lat = parseFloat( lat ); + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_lon = function(test, common) { + var lons = { + invalid: [ -360, -181, 181, 360 ], + valid: [ -180, -1, -0, 0, 45, 90, '-180', '0', '180' ] + }; + lons.invalid.forEach( function( lon ){ + test('invalid lon', function(t) { + sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){ + t.equal(err, 'invalid param \'lon\': must be >-180 and <180', 'invalid longitude'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + lons.valid.forEach( function( lon ){ + test('valid lon', function(t) { + sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.lon = parseFloat( lon ); + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_zoom = function(test, common) { + test('invalid zoom value', function(t) { + sanitize({ zoom: 'a', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 10, 'default zoom set'); + t.end(); + }); + }); + test('below min zoom value', function(t) { + sanitize({ zoom: -100, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 1, 'min zoom set'); + t.end(); + }); + }); + test('above max zoom value', function(t) { + sanitize({ zoom: 9999, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 18, 'max zoom set'); + t.end(); + }); + }); +}; + +module.exports.tests.sanitize_size = function(test, common) { + test('invalid size value', function(t) { + sanitize({ size: 'a', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 10, 'default size set'); + t.end(); + }); + }); + test('below min size value', function(t) { + sanitize({ size: -100, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 1, 'min size set'); + t.end(); + }); + }); + test('above max size value', function(t) { + sanitize({ size: 9999, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 40, 'max size set'); + t.end(); + }); + }); +}; + +module.exports.tests.sanitize_layers = function(test, common) { + test('unspecified', function(t) { + sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.deepEqual(clean.layers, defaultClean.layers, 'default layers set'); + t.end(); + }); + }); + test('invalid layer', function(t) { + sanitize({ layers: 'test_layer', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + var msg = 'invalid param \'layer\': must be one or more of geoname,osmnode,osmway,admin0,admin1,admin2,neighborhood'; + t.equal(err, msg, 'invalid layer requested'); + t.end(); + }); + }); +}; + +module.exports.tests.invalid_params = function(test, common) { + test('invalid input params', function(t) { + sanitize( undefined, function( err, clean ){ + t.equal(err, defaultError, 'handle invalid params gracefully'); + t.end(); + }); + }); +}; + +module.exports.tests.middleware_failure = function(test, common) { + test('middleware failure', function(t) { + var res = { status: function( code ){ + t.equal(code, 400, 'status set'); + }}; + var next = function( message ){ + t.equal(message, defaultError); + t.end(); + }; + sanitize.middleware( {}, res, next ); + }); +}; + +module.exports.tests.middleware_success = function(test, common) { + test('middleware success', function(t) { + var req = { query: { input: 'test', lat: 0, lon: 0 }}; + var next = function( message ){ + t.equal(message, undefined, 'no error message set'); + t.deepEqual(req.clean, defaultClean); + t.end(); + }; + sanitize.middleware( req, undefined, next ); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('SANTIZE /sanitise ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file