From 0f0bd3e4ce2fd9132c0083682b70a3e909034ad3 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Mon, 1 May 2017 09:42:08 -0400 Subject: [PATCH] added initial attempt at generic http json service --- service/http_json.js | 91 +++++++ test/unit/run.js | 1 + test/unit/service/http_json.js | 435 +++++++++++++++++++++++++++++++++ 3 files changed, 527 insertions(+) create mode 100644 service/http_json.js create mode 100644 test/unit/service/http_json.js diff --git a/service/http_json.js b/service/http_json.js new file mode 100644 index 00000000..ac1db802 --- /dev/null +++ b/service/http_json.js @@ -0,0 +1,91 @@ +const request = require('request'); +const bl = require('bl'); +const _ = require('lodash'); +const isDNT = require( '../helper/logging' ).isDNT; + +const logger = require( 'pelias-logger' ).get( 'placeholder' ); + +module.exports = function setup(serviceConfig) { + if (!_.conformsTo(serviceConfig, { + getName: _.isFunction, + getBaseUrl: _.isFunction, + getUrl: _.isFunction, + getParameters: _.isFunction, + getHeaders: _.isFunction + })) { + throw Error('serviceConfig should have a bunch of functions exposed'); + } + + if (_.isEmpty(serviceConfig.getBaseUrl())) { + logger.warn(`${serviceConfig.getName()} service disabled`); + + return (req, callback) => { + // respond with an error to any call + callback(`${serviceConfig.getName()} service disabled`); + }; + + } + + logger.info(`using ${serviceConfig.getName()} service at ${serviceConfig.getBaseUrl()}`); + return (req, callback) => { + const options = { + method: 'GET', + url: serviceConfig.getUrl(req), + qs: serviceConfig.getParameters(req), + headers: serviceConfig.getHeaders(req) + }; + + const do_not_track = isDNT(req); + + if (do_not_track) { + options.headers.dnt = '1'; + } + + request(options).on('response', (response) => { + // pipe the response thru bl which will accumulate the entire body + response.pipe(bl((err, data) => { + if (response.statusCode === 200) { + // parse and return w/o error unless response wasn't JSON + try { + const parsed = JSON.parse(data); + return callback(null, parsed); + + } + catch (err) { + if (do_not_track) { + logger.error(`${serviceConfig.getBaseUrl()} [do_not_track] could not parse response: ${data}`); + return callback(`${serviceConfig.getBaseUrl()} [do_not_track] could not parse response: ${data}`); + } else { + logger.error(`${response.request.href} could not parse response: ${data}`); + return callback(`${response.request.href} could not parse response: ${data}`); + } + + } + } + else { + // otherwise there was a non-200 status so handle generically + if (do_not_track) { + logger.error(`${serviceConfig.getBaseUrl()} [do_not_track] returned status ${response.statusCode}: ${data}`); + return callback(`${serviceConfig.getBaseUrl()} [do_not_track] returned status ${response.statusCode}: ${data}`); + } else { + logger.error(`${response.request.href} returned status ${response.statusCode}: ${data}`); + return callback(`${response.request.href} returned status ${response.statusCode}: ${data}`); + } + + } + })); + + }) + .on('error', (err) => { + if (do_not_track) { + logger.error(`${serviceConfig.getBaseUrl()} [do_not_track]: ${JSON.stringify(err)}`); + callback(err); + } else { + logger.error(`${options.url}: ${JSON.stringify(err)}`); + callback(err); + } + }); + + }; + +}; diff --git a/test/unit/run.js b/test/unit/run.js index f5638dba..4fb1e5a2 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -80,6 +80,7 @@ var tests = [ require('./sanitizer/search'), require('./sanitizer/search_fallback'), require('./sanitizer/wrap'), + require('./service/http_json'), require('./service/mget'), require('./service/search'), require('./service/interpolation'), diff --git a/test/unit/service/http_json.js b/test/unit/service/http_json.js new file mode 100644 index 00000000..98de94ce --- /dev/null +++ b/test/unit/service/http_json.js @@ -0,0 +1,435 @@ +const proxyquire = require('proxyquire').noCallThru(); +const express = require('express'); + +const setup = require('../../../service/http_json'); + +module.exports.tests = {}; + +module.exports.tests.interface = (test, common) => { + test('valid interface', (t) => { + const logger = require('pelias-mock-logger')(); + + var service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + }); + + t.equal(typeof service, 'function', 'service is a function'); + t.end(); + }); +}; + +module.exports.tests.conforms_to = (test, common) => { + test('serviceConfig with non-function getName property should throw error', (t) => { + const serviceConfig = { + getName: 'this is not a function', + getBaseUrl: () => {}, + getUrl: () => {}, + getParameters: () => {}, + getHeaders: () => {} + }; + + t.throws(setup.bind(null, serviceConfig), /serviceConfig should have a bunch of functions exposed/); + t.end(); + + }); + + test('serviceConfig with non-function getBaseUrl property should throw error', (t) => { + const serviceConfig = { + getName: () => {}, + getBaseUrl: 'this is not a function', + getUrl: () => {}, + getParameters: () => {}, + getHeaders: () => {} + }; + + t.throws(setup.bind(null, serviceConfig), /serviceConfig should have a bunch of functions exposed/); + t.end(); + + }); + + test('serviceConfig with non-function getUrl property should throw error', (t) => { + const serviceConfig = { + getName: () => {}, + getBaseUrl: () => {}, + getUrl: 'this is not a function', + getParameters: () => {}, + getHeaders: () => {} + }; + + t.throws(setup.bind(null, serviceConfig), /serviceConfig should have a bunch of functions exposed/); + t.end(); + + }); + + test('serviceConfig with non-function getParameters property should throw error', (t) => { + const serviceConfig = { + getName: () => {}, + getBaseUrl: () => {}, + getUrl: () => {}, + getParameters: 'this is not a function', + getHeaders: () => {} + }; + + t.throws(setup.bind(null, serviceConfig), /serviceConfig should have a bunch of functions exposed/); + t.end(); + + }); + + test('serviceConfig with non-function getHeaders property should throw error', (t) => { + const serviceConfig = { + getName: () => {}, + getBaseUrl: () => {}, + getUrl: () => {}, + getParameters: () => {}, + getHeaders: 'this is not a function' + }; + + t.throws(setup.bind(null, serviceConfig), /serviceConfig should have a bunch of functions exposed/); + t.end(); + + }); + +}; + +module.exports.tests.do_nothing_service = (test, common) => { + test('undefined config.url should return service that logs that config.name service is not available', (t) => { + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { + return undefined; + }, + getUrl: () => { return undefined; }, + getParameters: (req) => {}, + getHeaders: (req) => {} + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + })(serviceConfig); + + t.ok(logger.isWarnMessage(/^foo service disabled$/)); + + service({}, (err) => { + t.equals(err, 'foo service disabled'); + t.end(); + }); + + }); + +}; + +module.exports.tests.failure_conditions = (test, common) => { + test('server returning error should log it and return no results', (t) => { + const server = express().listen(); + const port = server.address().port; + + // immediately close the server so to ensure an error response + server.close(); + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/built_url`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err.code, 'ECONNREFUSED'); + t.notOk(results); + t.ok(logger.isErrorMessage(new RegExp(`^http://localhost:${port}/built_url: .*ECONNREFUSED`)), + 'there should be a connection refused error message'); + t.end(); + + server.close(); + + }); + + }); + + test('[DNT] server returning error should log it w/sanitized URL and return no results', (t) => { + const server = express().listen(); + const port = server.address().port; + + // immediately close the server so to ensure an error response + server.close(); + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/built_url`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger, + '../helper/logging': { + isDNT: () => { return true; } + } + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err.code, 'ECONNREFUSED'); + t.notOk(results); + t.ok(logger.isErrorMessage(new RegExp(`^http://localhost:${port} \\[do_not_track\\]: .*ECONNREFUSED`)), + 'there should be a connection refused error message'); + t.end(); + + server.close(); + + }); + + }); + + test('server returning non-200 response should log error and return no results', (t) => { + const webServer = express(); + webServer.get('/some_endpoint', (req, res, next) => { + t.notOk(req.headers.hasOwnProperty('dnt'), 'dnt header should not have been passed'); + + t.equals(req.headers.header1, 'header1 value', 'all headers should have been passed'); + t.deepEquals(req.query, { param1: 'param1 value', param2: 'param2 value' }); + + res.status(400).send('a bad request was made'); + }); + + const server = webServer.listen(); + const port = server.address().port; + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/some_endpoint`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err, `http://localhost:${port}/some_endpoint?param1=param1%20value¶m2=param2%20value ` + + 'returned status 400: a bad request was made'); + t.notOk(results); + t.ok(logger.isErrorMessage(`http://localhost:${port}/some_endpoint?param1=param1%20value¶m2=param2%20value ` + + `returned status 400: a bad request was made`)); + t.end(); + + server.close(); + + }); + + }); + + test('[DNT] server returning non-200 response should log sanitized error and return no results', (t) => { + const webServer = express(); + webServer.get('/some_endpoint', (req, res, next) => { + t.equals(req.headers.dnt, '1'); + + t.equals(req.headers.header1, 'header1 value', 'all headers should have been passed'); + t.deepEquals(req.query, { param1: 'param1 value', param2: 'param2 value' }); + + res.status(400).send('a bad request was made'); + }); + + const server = webServer.listen(); + const port = server.address().port; + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/some_endpoint`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger, + '../helper/logging': { + isDNT: () => { return true; } + } + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err, `http://localhost:${port} [do_not_track] returned status 400: a bad request was made`); + t.notOk(results); + t.ok(logger.isErrorMessage(`http://localhost:${port} [do_not_track] ` + + `returned status 400: a bad request was made`)); + t.end(); + + server.close(); + + }); + + }); + + test('server returning 200 statusCode but with non-JSON response should log error and return undefined', (t) => { + const webServer = express(); + webServer.get('/some_endpoint', (req, res, next) => { + t.notOk(req.headers.hasOwnProperty('dnt'), 'dnt header should not have been passed'); + + t.equals(req.headers.header1, 'header1 value', 'all headers should have been passed'); + t.deepEquals(req.query, { param1: 'param1 value', param2: 'param2 value' }); + + res.status(200).send('this is not parseable as JSON'); + }); + + const server = webServer.listen(); + const port = server.address().port; + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/some_endpoint`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err, `http://localhost:${port}/some_endpoint?param1=param1%20value¶m2=param2%20value ` + + `could not parse response: this is not parseable as JSON`); + t.notOk(results, 'should return undefined'); + t.ok(logger.isErrorMessage(`http://localhost:${port}/some_endpoint?param1=param1%20value¶m2=param2%20value ` + + `could not parse response: this is not parseable as JSON`)); + t.end(); + + server.close(); + + }); + + }); + + test('[DNT] server returning 200 statusCode but with non-JSON response should log sanitized error and return undefined', (t) => { + const webServer = express(); + webServer.get('/some_endpoint', (req, res, next) => { + t.equals(req.headers.dnt, '1'); + + t.equals(req.headers.header1, 'header1 value', 'all headers should have been passed'); + t.deepEquals(req.query, { param1: 'param1 value', param2: 'param2 value' }); + + res.status(200).send('this is not parseable as JSON'); + }); + + const server = webServer.listen(); + const port = server.address().port; + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/some_endpoint`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger, + '../helper/logging': { + isDNT: () => { return true; } + } + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.equals(err, `http://localhost:${port} [do_not_track] ` + + `could not parse response: this is not parseable as JSON`); + t.notOk(results, 'should return undefined'); + t.ok(logger.isErrorMessage(`http://localhost:${port} [do_not_track] ` + + `could not parse response: this is not parseable as JSON`)); + t.end(); + + server.close(); + + }); + + }); + +}; + +module.exports.tests.success_conditions = (test, common) => { + test('server returning statusCode 200 should return no error and parsed output', (t) => { + const webServer = express(); + webServer.get('/some_endpoint', (req, res, next) => { + t.notOk(req.headers.hasOwnProperty('dnt'), 'dnt header should not have been passed'); + + t.equals(req.headers.header1, 'header1 value', 'all headers should have been passed'); + t.deepEquals(req.query, { param1: 'param1 value', param2: 'param2 value' }); + + res.status(200).send('[1, 2, 3]'); + }); + + const server = webServer.listen(); + const port = server.address().port; + + const logger = require('pelias-mock-logger')(); + + const serviceConfig = { + getName: () => { return 'foo'; }, + getBaseUrl: () => { return `http://localhost:${port}`; }, + getUrl: (req) => { return `http://localhost:${port}/some_endpoint`; }, + getParameters: (req) => { return { param1: 'param1 value', param2: 'param2 value' }; }, + getHeaders: (req) => { return { header1: 'header1 value' }; } + }; + + const service = proxyquire('../../../service/http_json', { + 'pelias-logger': logger + })(serviceConfig); + + t.ok(logger.isInfoMessage(new RegExp(`using foo service at http://localhost:${port}`))); + + service({}, (err, results) => { + t.notOk(err, 'should be no error'); + t.deepEquals(results, [1, 2, 3]); + t.notOk(logger.hasErrorMessages()); + t.end(); + + server.close(); + + }); + + }); + +}; + +module.exports.all = (tape, common) => { + function test(name, testFunction) { + return tape(`SERVICE /http_json ${name}`, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +};