From 794db085a26181ee85ea5ef06b022ce91a19b76b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 12:18:56 +0100 Subject: [PATCH 01/10] split app from server, add toobusy --- app.js | 24 ++++++++++++++++++++++++ index.js | 26 ++++++-------------------- middleware/toobusy.js | 19 +++++++++++++++++++ package.json | 3 ++- 4 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 app.js create mode 100644 middleware/toobusy.js diff --git a/app.js b/app.js new file mode 100644 index 00000000..30761954 --- /dev/null +++ b/app.js @@ -0,0 +1,24 @@ + +var app = require('express')(); + +/** ----------------------- middleware ----------------------- **/ + +app.use( require('./middleware/toobusy') ); // should be first +app.use( require('./middleware/headers') ); +app.use( require('./middleware/cors') ); +app.use( require('./middleware/jsonp') ); + +/** ----------------------- routes ----------------------- **/ + +// api root +app.get( '/', require('./controller/index') ); + +// suggest API +app.get( '/suggest', require('./sanitiser/suggest'), require('./controller/suggest') ); + +/** ----------------------- error middleware ----------------------- **/ + +app.use( require('./middleware/404') ); +app.use( require('./middleware/500') ); + +app.listen( process.env.PORT || 3100 ); \ No newline at end of file diff --git a/index.js b/index.js index e74fed10..e788c86b 100644 --- a/index.js +++ b/index.js @@ -1,23 +1,9 @@ -var app = require('express')(); +/** cluster webserver across all cores **/ -/** ----------------------- middleware ----------------------- **/ +var cluster = require('cluster'), + app = require('./app'); -app.use( require('./middleware/headers') ); -app.use( require('./middleware/cors') ); -app.use( require('./middleware/jsonp') ); - -/** ----------------------- routes ----------------------- **/ - -// api root -app.get( '/', require('./controller/index') ); - -// suggest API -app.get( '/suggest', require('./sanitiser/suggest'), require('./controller/suggest') ); - -/** ----------------------- error middleware ----------------------- **/ - -app.use( require('./middleware/404') ); -app.use( require('./middleware/500') ); - -app.listen( process.env.PORT || 3100 ); \ No newline at end of file +cluster(app) + .use(cluster.stats()) + .listen( process.env.PORT || 3100 ); \ No newline at end of file diff --git a/middleware/toobusy.js b/middleware/toobusy.js new file mode 100644 index 00000000..bbe6784f --- /dev/null +++ b/middleware/toobusy.js @@ -0,0 +1,19 @@ + +// middleware which blocks requests when the eventloop is too busy +var toobusy = require('toobusy'); + +function middleware(req, res, next){ + if( toobusy() ){ + res.status(503); // Service Unavailable + return next('Server Overwhelmed'); + } + return next(); +} + +// calling .shutdown allows your process to exit normally +process.on('SIGINT', function() { + toobusy.shutdown(); + process.exit(); +}); + +module.exports = middleware; \ No newline at end of file diff --git a/package.json b/package.json index 46c15a5d..32e98b8a 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "dependencies": { "express": "^4.8.8", "geopipes-elasticsearch-backend": "0.0.7", - "pelias-esclient": "0.0.25" + "pelias-esclient": "0.0.25", + "toobusy": "^0.2.4" }, "devDependencies": { "ciao": "^0.3.4", From 18f4f6b25642257f76428a25b2e6a82f89a9172b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 13:30:21 +0100 Subject: [PATCH 02/10] refactor/ add more tests --- app.js | 6 ++-- controller/index.js | 24 ++++++++----- controller/suggest.js | 52 +++++++++++++++------------ index.js | 20 +++++++---- src/backend.js | 11 ++---- test/unit/controller/index.js | 6 ++-- test/unit/controller/suggest.js | 64 +++++++++++++++++++++++++++++++++ test/unit/mock/backend.js | 28 +++++++++++++++ test/unit/mock/query.js | 10 ++++++ test/unit/run.js | 3 +- 10 files changed, 171 insertions(+), 53 deletions(-) create mode 100644 test/unit/controller/suggest.js create mode 100644 test/unit/mock/backend.js create mode 100644 test/unit/mock/query.js diff --git a/app.js b/app.js index 30761954..a951f06b 100644 --- a/app.js +++ b/app.js @@ -11,14 +11,14 @@ app.use( require('./middleware/jsonp') ); /** ----------------------- routes ----------------------- **/ // api root -app.get( '/', require('./controller/index') ); +app.get( '/', require('./controller/index')() ); // suggest API -app.get( '/suggest', require('./sanitiser/suggest'), require('./controller/suggest') ); +app.get( '/suggest', require('./sanitiser/suggest'), require('./controller/suggest')() ); /** ----------------------- error middleware ----------------------- **/ app.use( require('./middleware/404') ); app.use( require('./middleware/500') ); -app.listen( process.env.PORT || 3100 ); \ No newline at end of file +module.exports = app; \ No newline at end of file diff --git a/controller/index.js b/controller/index.js index 6936a982..10f2d9a4 100644 --- a/controller/index.js +++ b/controller/index.js @@ -1,16 +1,22 @@ var pkg = require('../package'); -function controller( req, res, next ){ +function setup(){ - // stats - res.json({ - name: pkg.name, - version: { - number: pkg.version - } - }); + function controller( req, res, next ){ + + // stats + res.json({ + name: pkg.name, + version: { + number: pkg.version + } + }); + + } + + return controller; } -module.exports = controller; \ No newline at end of file +module.exports = setup; \ No newline at end of file diff --git a/controller/suggest.js b/controller/suggest.js index c6967186..b62b2fe6 100644 --- a/controller/suggest.js +++ b/controller/suggest.js @@ -1,35 +1,41 @@ -var query = require('../query/suggest'), - backend = require('../src/backend'); +function setup( backend, query ){ -function controller( req, res, next ){ + // allow overriding of dependencies + backend = backend || require('../src/backend'); + query = query || require('../query/suggest'); - // backend command - var cmd = { - index: 'pelias', - body: query( req.clean ) - }; + function controller( req, res, next ){ - // query backend - backend().client.suggest( cmd, function( err, data ){ + // backend command + var cmd = { + index: 'pelias', + body: query( req.clean ) + }; - var docs = []; + // query backend + backend().client.suggest( cmd, function( err, data ){ - // handle backend errors - if( err ){ return next( err ); } + var docs = []; - // map response to a valid FeatureCollection - if( data && Array.isArray( data.pelias ) && data.pelias.length ){ - docs = data['pelias'][0].options || []; - } + // handle backend errors + if( err ){ return next( err ); } - // respond - return res.status(200).json({ - date: new Date().getTime(), - body: docs + // map response to a valid FeatureCollection + if( data && Array.isArray( data.pelias ) && data.pelias.length ){ + docs = data['pelias'][0].options || []; + } + + // respond + return res.status(200).json({ + date: new Date().getTime(), + body: docs + }); }); - }); + } + + return controller; } -module.exports = controller; \ No newline at end of file +module.exports = setup; \ No newline at end of file diff --git a/index.js b/index.js index e788c86b..3003c068 100644 --- a/index.js +++ b/index.js @@ -1,9 +1,17 @@ -/** cluster webserver across all cores **/ - var cluster = require('cluster'), - app = require('./app'); + app = require('./app'), + multicore = false, + port = ( process.env.PORT || 3100 ); -cluster(app) - .use(cluster.stats()) - .listen( process.env.PORT || 3100 ); \ No newline at end of file +/** cluster webserver across all cores **/ +if( multicore ){ + // @todo: not finished yet + // cluster(app) + // .use(cluster.stats()) + // .listen( process.env.PORT || 3100 ); +} +else { + console.log( 'listening on ' + port ); + app.listen( process.env.PORT || 3100 ); +} \ No newline at end of file diff --git a/src/backend.js b/src/backend.js index 2a9d60ae..d0388966 100644 --- a/src/backend.js +++ b/src/backend.js @@ -1,14 +1,7 @@ var Backend = require('geopipes-elasticsearch-backend'), - backends = {}, - client; - -// set env specific client -if( process.env.NODE_ENV === 'test' ){ - client = require('./pelias-mockclient'); -} else { - client = require('pelias-esclient')(); -} + client = require('pelias-esclient')(), + backends = {}; function getBackend( index, type ){ var key = ( index + ':' + type ); diff --git a/test/unit/controller/index.js b/test/unit/controller/index.js index ef1d139d..e9752511 100644 --- a/test/unit/controller/index.js +++ b/test/unit/controller/index.js @@ -1,17 +1,19 @@ -var controller = require('../../../controller/index'); +var setup = require('../../../controller/index'); module.exports.tests = {}; module.exports.tests.interface = function(test, common) { test('valid interface', function(t) { - t.equal(typeof controller, 'function', 'controller is a function'); + t.equal(typeof setup, 'function', 'setup is a function'); + t.equal(typeof setup(), 'function', 'setup returns a controller'); t.end(); }); }; module.exports.tests.info = function(test, common) { test('returns server info', function(t) { + var controller = setup(); var res = { json: function( json ){ t.equal(typeof json, 'object', 'returns json'); t.equal(typeof json.name, 'string', 'name'); diff --git a/test/unit/controller/suggest.js b/test/unit/controller/suggest.js new file mode 100644 index 00000000..02c40b43 --- /dev/null +++ b/test/unit/controller/suggest.js @@ -0,0 +1,64 @@ + +var setup = require('../../../controller/suggest'), + mockBackend = require('../mock/backend'), + mockQuery = require('../mock/query'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof setup, 'function', 'setup is a function'); + t.equal(typeof setup(), 'function', 'setup returns a controller'); + t.end(); + }); +}; + +// functionally test controller (backend success) +module.exports.tests.functional_success = function(test, common) { + test('functional test', function(t) { + var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){ + t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command'); + }); + var controller = setup( backend, mockQuery() ); + var res = { + status: function( code ){ + t.equal(code, 200, 'status set'); + return res; + }, + json: function( json ){ + t.equal(typeof json, 'object', 'returns json'); + t.equal(typeof json.date, 'number', 'date set'); + t.true(Array.isArray(json.body), 'body is array'); + t.deepEqual(json.body, [ { value: 1 }, { value: 2 } ], 'values correctly mapped'); + t.end(); + } + }; + controller( { clean: { a: 'b' } }, res ); + }); +}; + +// functionally test controller (backend failure) +module.exports.tests.functional_failure = function(test, common) { + test('functional test', function(t) { + var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){ + t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command'); + }); + var controller = setup( backend, mockQuery() ); + var next = function( message ){ + t.equal(message,'a backend error occurred','error passed to errorHandler'); + t.end(); + }; + controller( { clean: { a: 'b' } }, undefined, next ); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('GET /suggest ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js new file mode 100644 index 00000000..8d87fedc --- /dev/null +++ b/test/unit/mock/backend.js @@ -0,0 +1,28 @@ + +var responses = {}; +responses['client/suggest/ok/1'] = function( cmd, cb ){ + return cb( undefined, suggestEnvelope([ { value: 1 }, { value: 2 } ]) ); +}; +responses['client/suggest/fail/1'] = function( cmd, cb ){ + return cb( 'a backend error occurred' ); +}; + +function setup( key, cmdCb ){ + function backend( a, b ){ + return { + client: { + suggest: function( cmd, cb ){ + if( 'function' === typeof cmdCb ){ cmdCb( cmd ); } + return responses[key].apply( this, arguments ); + } + } + }; + } + return backend; +} + +function suggestEnvelope( options ){ + return { pelias: [{ options: options }]}; +} + +module.exports = setup; \ No newline at end of file diff --git a/test/unit/mock/query.js b/test/unit/mock/query.js new file mode 100644 index 00000000..2a5f21fd --- /dev/null +++ b/test/unit/mock/query.js @@ -0,0 +1,10 @@ + +function setup(){ + return query; +} + +function query( clean ){ + return clean; +} + +module.exports = setup; \ No newline at end of file diff --git a/test/unit/run.js b/test/unit/run.js index 36045ca9..50bfccfd 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -3,7 +3,8 @@ var tape = require('tape'); var common = {}; var tests = [ - require('./controller/index') + require('./controller/index'), + require('./controller/suggest') ]; tests.map(function(t) { From 42a2387003f8ab8ecd519339753de8be60420aec Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 13:43:59 +0100 Subject: [PATCH 03/10] tests --- app.js | 2 +- sanitiser/suggest.js | 5 ++- test/unit/run.js | 3 +- test/unit/sanitiser/suggest.js | 57 ++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 test/unit/sanitiser/suggest.js diff --git a/app.js b/app.js index a951f06b..842cb385 100644 --- a/app.js +++ b/app.js @@ -14,7 +14,7 @@ app.use( require('./middleware/jsonp') ); app.get( '/', require('./controller/index')() ); // suggest API -app.get( '/suggest', require('./sanitiser/suggest'), require('./controller/suggest')() ); +app.get( '/suggest', require('./sanitiser/suggest').middleware, require('./controller/suggest')() ); /** ----------------------- error middleware ----------------------- **/ diff --git a/sanitiser/suggest.js b/sanitiser/suggest.js index d7dfa60a..a2112965 100644 --- a/sanitiser/suggest.js +++ b/sanitiser/suggest.js @@ -68,8 +68,11 @@ function sanitize( params, cb ){ } +// export function +module.exports = sanitize; + // middleware -module.exports = function( req, res, next ){ +module.exports.middleware = function( req, res, next ){ sanitize( req.query, function( err, clean ){ if( err ){ res.status(400); // 400 Bad Request diff --git a/test/unit/run.js b/test/unit/run.js index 50bfccfd..ffc77c68 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -4,7 +4,8 @@ var common = {}; var tests = [ require('./controller/index'), - require('./controller/suggest') + require('./controller/suggest'), + require('./sanitiser/suggest') ]; tests.map(function(t) { diff --git a/test/unit/sanitiser/suggest.js b/test/unit/sanitiser/suggest.js new file mode 100644 index 00000000..6c76a605 --- /dev/null +++ b/test/unit/sanitiser/suggest.js @@ -0,0 +1,57 @@ + +var sanitize = require('../../../sanitiser/suggest'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('sanitize interface', function(t) { + t.equal(typeof sanitize, 'function', 'sanitize is a function'); + t.equal(sanitize.length, 2, 'sanitize interface'); + t.end(); + }); + test('middleware interface', function(t) { + t.equal(typeof sanitize.middleware, 'function', 'middleware is a function'); + t.equal(sanitize.middleware.length, 3, 'sanitize is valid middleware'); + t.end(); + }); +}; + +module.exports.tests.middleware_failure = function(test, common) { + test('middleware failure', function(t) { + var res = { status: function( code ){ + t.equal(code, 400, 'status set'); + }}; + var next = function( message ){ + t.equal(message,'invalid param \'input\': text length, must be >0'); + t.end(); + }; + sanitize.middleware( {}, res, next ); + }); +}; + +module.exports.tests.middleware_success = function(test, common) { + test('middleware success', function(t) { + var req = { query: { input: 'test', lat: 0, lon: 0 }}; + var next = function( message ){ + t.equal(message, undefined, 'no error message set'); + t.deepEqual(req.clean, { + input: 'test', lat: 0, lon: 0, + layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood' ], + size: 10, zoom: 10 + }); + t.end(); + }; + sanitize.middleware( req, undefined, next ); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('SANTIZE /suggest ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file From ffda37adffd7bfee3beec9dc8c30d6235bfd3501 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 14:18:06 +0100 Subject: [PATCH 04/10] tests --- sanitiser/suggest.js | 6 +- test/unit/sanitiser/suggest.js | 160 +++++++++++++++++++++++++++++++-- 2 files changed, 156 insertions(+), 10 deletions(-) diff --git a/sanitiser/suggest.js b/sanitiser/suggest.js index a2112965..85f972f6 100644 --- a/sanitiser/suggest.js +++ b/sanitiser/suggest.js @@ -21,7 +21,7 @@ function sanitize( params, cb ){ // total results var size = parseInt( params.size, 10 ); if( !isNaN( size ) ){ - clean.size = Math.min( size, 40 ); // max + clean.size = Math.min( Math.max( size, 1 ), 40 ); // max } else { clean.size = 10; // default } @@ -33,7 +33,7 @@ function sanitize( params, cb ){ }); for( var x=0; x0', + defaultClean = { input: 'test', lat: 0, layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood' ], lon: 0, size: 10, zoom: 10 }; module.exports.tests = {}; @@ -16,13 +18,161 @@ module.exports.tests.interface = function(test, common) { }); }; +module.exports.tests.sanitize_input = function(test, common) { + var inputs = { + invalid: [ '', 100, null, undefined, new Date() ], + valid: [ 'a', 'aa', 'aaaaaaaa' ] + }; + inputs.invalid.forEach( function( input ){ + test('invalid input', function(t) { + sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){ + t.equal(err, 'invalid param \'input\': text length, must be >0', 'invalid input'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + inputs.valid.forEach( function( input ){ + test('valid input', function(t) { + sanitize({ input: input, lat: 0, lon: 0 }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.input = input; + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_lat = function(test, common) { + var lats = { + invalid: [ -1, -45, -90, 91, 120, 181 ], + valid: [ 0, 45, 90, -0, '0', '45', '90' ] + }; + lats.invalid.forEach( function( lat ){ + test('invalid lat', function(t) { + sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){ + t.equal(err, 'invalid param \'lat\': must be >0 and <90', 'invalid latitude'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + lats.valid.forEach( function( lat ){ + test('valid lat', function(t) { + sanitize({ input: 'test', lat: lat, lon: 0 }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.lat = parseFloat( lat ); + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_lon = function(test, common) { + var lons = { + invalid: [ -360, -181, 181, 360 ], + valid: [ -180, -1, -0, 0, 45, 90, '-180', '0', '180' ] + }; + lons.invalid.forEach( function( lon ){ + test('invalid lon', function(t) { + sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){ + t.equal(err, 'invalid param \'lon\': must be >-180 and <180', 'invalid longitude'); + t.equal(clean, undefined, 'clean not set'); + t.end(); + }); + }); + }); + lons.valid.forEach( function( lon ){ + test('valid lon', function(t) { + sanitize({ input: 'test', lat: 0, lon: lon }, function( err, clean ){ + var expected = JSON.parse(JSON.stringify( defaultClean )); + expected.lon = parseFloat( lon ); + t.equal(err, undefined, 'no error'); + t.deepEqual(clean, expected, 'clean set correctly'); + t.end(); + }); + }); + }); +}; + +module.exports.tests.sanitize_zoom = function(test, common) { + test('invalid zoom value', function(t) { + sanitize({ zoom: 'a', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 10, 'default zoom set'); + t.end(); + }); + }); + test('below min zoom value', function(t) { + sanitize({ zoom: -100, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 1, 'min zoom set'); + t.end(); + }); + }); + test('above max zoom value', function(t) { + sanitize({ zoom: 9999, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.zoom, 18, 'max zoom set'); + t.end(); + }); + }); +}; + +module.exports.tests.sanitize_size = function(test, common) { + test('invalid size value', function(t) { + sanitize({ size: 'a', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 10, 'default size set'); + t.end(); + }); + }); + test('below min size value', function(t) { + sanitize({ size: -100, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 1, 'min size set'); + t.end(); + }); + }); + test('above max size value', function(t) { + sanitize({ size: 9999, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.equal(clean.size, 40, 'max size set'); + t.end(); + }); + }); +}; + +module.exports.tests.sanitize_layers = function(test, common) { + test('unspecified', function(t) { + sanitize({ layers: undefined, input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + t.deepEqual(clean.layers, defaultClean.layers, 'default layers set'); + t.end(); + }); + }); + test('invalid layer', function(t) { + sanitize({ layers: 'test_layer', input: 'test', lat: 0, lon: 0 }, function( err, clean ){ + var msg = 'invalid param \'layer\': must be one or more of geoname,osmnode,osmway,admin0,admin1,admin2,neighborhood'; + t.equal(err, msg, 'invalid layer requested'); + t.end(); + }); + }); +}; + +module.exports.tests.invalid_params = function(test, common) { + test('invalid input params', function(t) { + sanitize( undefined, function( err, clean ){ + t.equal(err, defaultError, 'handle invalid params gracefully'); + t.end(); + }); + }); +}; + module.exports.tests.middleware_failure = function(test, common) { test('middleware failure', function(t) { var res = { status: function( code ){ t.equal(code, 400, 'status set'); }}; var next = function( message ){ - t.equal(message,'invalid param \'input\': text length, must be >0'); + t.equal(message, defaultError); t.end(); }; sanitize.middleware( {}, res, next ); @@ -34,11 +184,7 @@ module.exports.tests.middleware_success = function(test, common) { var req = { query: { input: 'test', lat: 0, lon: 0 }}; var next = function( message ){ t.equal(message, undefined, 'no error message set'); - t.deepEqual(req.clean, { - input: 'test', lat: 0, lon: 0, - layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood' ], - size: 10, zoom: 10 - }); + t.deepEqual(req.clean, defaultClean); t.end(); }; sanitize.middleware( req, undefined, next ); From 3cd93315ea5763b9912e6357b95ef071bbdd5bb1 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 14:21:38 +0100 Subject: [PATCH 05/10] refactor --- app.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index 842cb385..d9659b17 100644 --- a/app.js +++ b/app.js @@ -8,13 +8,24 @@ app.use( require('./middleware/headers') ); app.use( require('./middleware/cors') ); app.use( require('./middleware/jsonp') ); +/** ----------------------- sanitisers ----------------------- **/ + +var sanitisers = {}; +sanitisers.suggest = require('./sanitiser/suggest'); + +/** ----------------------- controllers ----------------------- **/ + +var controllers = {}; +controllers.index = require('./controller/index'); +controllers.suggest = require('./controller/suggest'); + /** ----------------------- routes ----------------------- **/ // api root -app.get( '/', require('./controller/index')() ); +app.get( '/', controllers.index() ); // suggest API -app.get( '/suggest', require('./sanitiser/suggest').middleware, require('./controller/suggest')() ); +app.get( '/suggest', sanitisers.suggest.middleware, controllers.suggest() ); /** ----------------------- error middleware ----------------------- **/ From 2fd67fcc58ad03352e007acf5050ef08c84f6c1b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 14:50:10 +0100 Subject: [PATCH 06/10] refactor; add tests --- query/suggest.js | 4 ++-- test/unit/query/indeces.js | 23 +++++++++++++++++++++++ test/unit/query/suggest.js | 29 +++++++++++++++++++++++++++++ test/unit/run.js | 4 +++- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 test/unit/query/indeces.js create mode 100644 test/unit/query/suggest.js diff --git a/query/suggest.js b/query/suggest.js index b54e2518..9ba2e5c8 100644 --- a/query/suggest.js +++ b/query/suggest.js @@ -3,7 +3,7 @@ var logger = require('../src/logger'); // Build pelias suggest query function generate( params ){ - + var cmd = { 'pelias' : { 'text' : params.input, @@ -23,7 +23,7 @@ function generate( params ){ } }; - logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); + // logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); return cmd; } diff --git a/test/unit/query/indeces.js b/test/unit/query/indeces.js new file mode 100644 index 00000000..d074f620 --- /dev/null +++ b/test/unit/query/indeces.js @@ -0,0 +1,23 @@ + +var indeces = require('../../../query/indeces'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.true(Array.isArray(indeces), 'valid array'); + t.equal(indeces.length, 7, 'valid array'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('indeces ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file diff --git a/test/unit/query/suggest.js b/test/unit/query/suggest.js new file mode 100644 index 00000000..ce7f8861 --- /dev/null +++ b/test/unit/query/suggest.js @@ -0,0 +1,29 @@ + +var query = require('../../../query/suggest'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof query, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof query, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('suggest query ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file diff --git a/test/unit/run.js b/test/unit/run.js index ffc77c68..d9b2041e 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -5,7 +5,9 @@ var common = {}; var tests = [ require('./controller/index'), require('./controller/suggest'), - require('./sanitiser/suggest') + require('./sanitiser/suggest'), + require('./query/indeces'), + require('./query/suggest') ]; tests.map(function(t) { From 05c5accba9e7803bfdfa86072034f52f43b7344f Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 14:51:31 +0100 Subject: [PATCH 07/10] add security audit scripts --- package.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index c9fb2fc1..308a1fd5 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "start": "node index.js", "test": "npm run unit && npm run ciao", "unit": "node test/unit/run.js | tap-spec", - "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao" + "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", + "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;" }, "repository": { "type": "git", @@ -37,6 +38,7 @@ "devDependencies": { "ciao": "^0.3.4", "tape": "^2.13.4", - "tap-spec": "^0.2.0" + "tap-spec": "^0.2.0", + "nsp": "^0.3.0" } -} +} \ No newline at end of file From 59a589b2d45a151762a21ab85005810b62d570a6 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 14:53:46 +0100 Subject: [PATCH 08/10] docs --- docs/404.md | 26 +++++++++++++------------- docs/cors.md | 8 ++++---- docs/index.md | 28 ++++++++++++++-------------- docs/jsonp.md | 8 ++++---- docs/suggest/success.md | 22 +++++++++++----------- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/docs/404.md b/docs/404.md index e683e480..df08c981 100644 --- a/docs/404.md +++ b/docs/404.md @@ -1,6 +1,6 @@ # invalid path -*Generated: Fri Sep 12 2014 20:51:44 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 404 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=300", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=300", "content-type": "application/json; charset=utf-8", "content-length": "35", "etag": "W/\"23-dfdfa185\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` @@ -39,19 +39,14 @@ Status: 404 ## Tests -### ✓ cache-control header correctly set -```javascript -response.should.have.header 'Cache-Control','public,max-age=300' -``` - -### ✓ content-type header correctly set +### ✓ not found ```javascript -response.should.have.header 'Content-Type','application/json; charset=utf-8' +response.statusCode.should.equal 404 ``` -### ✓ not found +### ✓ cache-control header correctly set ```javascript -response.statusCode.should.equal 404 +response.should.have.header 'Cache-Control','public,max-age=300' ``` ### ✓ should respond in json with server info @@ -61,3 +56,8 @@ should.exist json.error json.error.should.equal 'not found: invalid path' ``` +### ✓ content-type header correctly set +```javascript +response.should.have.header 'Content-Type','application/json; charset=utf-8' +``` + diff --git a/docs/cors.md b/docs/cors.md index 0df8dc00..e30c37b7 100644 --- a/docs/cors.md +++ b/docs/cors.md @@ -1,6 +1,6 @@ # cross-origin resource sharing -*Generated: Fri Sep 12 2014 20:51:44 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "50", "etag": "W/\"32-85536434\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` diff --git a/docs/index.md b/docs/index.md index 963ab240..b3eee35e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # api root -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "50", "etag": "W/\"32-85536434\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` @@ -42,6 +42,12 @@ Status: 200 ## Tests +### ✓ server header correctly set +```javascript +response.should.have.header 'Server' +response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/ +``` + ### ✓ content-type header correctly set ```javascript response.should.have.header 'Content-Type','application/json; charset=utf-8' @@ -57,20 +63,14 @@ response.statusCode.should.equal 200 response.should.have.header 'Cache-Control','public,max-age=60' ``` -### ✓ charset header correctly set -```javascript -response.should.have.header 'Charset','utf8' -``` - -### ✓ server header correctly set +### ✓ vanity header correctly set ```javascript -response.should.have.header 'Server' -response.headers.server.should.match /Pelias\/\d{1,2}\.\d{1,2}\.\d{1,2}/ +response.should.have.header 'X-Powered-By','mapzen' ``` -### ✓ vanity header correctly set +### ✓ charset header correctly set ```javascript -response.should.have.header 'X-Powered-By','mapzen' +response.should.have.header 'Charset','utf8' ``` ### ✓ should respond in json with server info diff --git a/docs/jsonp.md b/docs/jsonp.md index a67e5c40..382f3104 100644 --- a/docs/jsonp.md +++ b/docs/jsonp.md @@ -1,6 +1,6 @@ # jsonp -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,16 +18,16 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/javascript; charset=utf-8", "content-length": "57", "etag": "W/\"39-b8a2aba1\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` diff --git a/docs/suggest/success.md b/docs/suggest/success.md index abcc5db4..943b2bfd 100644 --- a/docs/suggest/success.md +++ b/docs/suggest/success.md @@ -1,6 +1,6 @@ # valid suggest query -*Generated: Fri Sep 12 2014 20:51:45 GMT+0100 (BST)* +*Generated: Thu Sep 18 2014 14:53:39 GMT+0100 (BST)* ## Request ```javascript { @@ -18,22 +18,22 @@ Status: 200 { "x-powered-by": "mapzen", "charset": "utf8", + "cache-control": "public,max-age=60", + "server": "Pelias/0.0.0", "access-control-allow-origin": "*", "access-control-allow-methods": "GET", "access-control-allow-headers": "X-Requested-With,content-type", "access-control-allow-credentials": "true", - "server": "Pelias/0.0.0", - "cache-control": "public,max-age=60", "content-type": "application/json; charset=utf-8", "content-length": "1248", - "etag": "W/\"o9NALcf9i0O3JoLO7pfqog==\"", - "date": "Fri, 12 Sep 2014 19:51:44 GMT", + "etag": "W/\"H7TGcVqWuu8sAb6aBhpd6A==\"", + "date": "Thu, 18 Sep 2014 13:53:39 GMT", "connection": "close" } ``` ```javascript { - "date": 1410551504928, + "date": 1411048419796, "body": [ { "text": "ACRELÂNDIA, Brazil", @@ -121,6 +121,11 @@ Status: 200 ## Tests +### ✓ 200 ok +```javascript +response.statusCode.should.equal 200 +``` + ### ✓ valid response ```javascript now = new Date().getTime() @@ -132,8 +137,3 @@ should.exist json.body json.body.should.be.instanceof Array ``` -### ✓ 200 ok -```javascript -response.statusCode.should.equal 200 -``` - From 0ef84c4ba5c81875775a1d9bddad1b2261300d18 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 15:00:16 +0100 Subject: [PATCH 09/10] query test --- test/unit/query/suggest.js | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/test/unit/query/suggest.js b/test/unit/query/suggest.js index ce7f8861..bb747c11 100644 --- a/test/unit/query/suggest.js +++ b/test/unit/query/suggest.js @@ -1,18 +1,39 @@ -var query = require('../../../query/suggest'); +var generate = require('../../../query/suggest'); module.exports.tests = {}; module.exports.tests.interface = function(test, common) { test('valid interface', function(t) { - t.equal(typeof query, 'function', 'valid function'); + t.equal(typeof generate, 'function', 'valid function'); t.end(); }); }; -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { - t.equal(typeof query, 'function', 'valid function'); +module.exports.tests.query = function(test, common) { + test('valid query', function(t) { + var query = generate({ + input: 'test', size: 10, + lat: 0, lon: 0, + layers: ['test'] + }); + var expected = { + pelias: { + text: 'test', + completion: { + field: 'suggest', + size: 10, + context: { + dataset: [ 'test' ], + location: { + precision: 2, + value: [ 0, 0 ] + } + } + } + } + }; + t.deepEqual(query, expected, 'valid suggest query'); t.end(); }); }; From 6eda46e41d3bb687136198b1f081dc3e41683de5 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 18 Sep 2014 15:01:42 +0100 Subject: [PATCH 10/10] drop node 11 from travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 607bac83..82ec939c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: node_js script: "npm run unit" node_js: - - "0.11" - "0.10" \ No newline at end of file