Browse Source

initialize esclient in routes/v1, eliminating need for src/backend

pull/782/head
Stephen Hess 8 years ago
parent
commit
8c249c26f4
  1. 4
      controller/place.js
  2. 8
      controller/search.js
  3. 19
      routes/v1.js
  4. 12
      service/mget.js
  5. 10
      service/search.js
  6. 6
      src/backend.js
  7. 2
      test/unit/controller/place.js
  8. 2
      test/unit/controller/search.js
  9. 28
      test/unit/mock/backend.js
  10. 5
      test/unit/run.js
  11. 4
      test/unit/service/mget.js
  12. 4
      test/unit/service/search.js
  13. 40
      test/unit/src/backend.js

4
controller/place.js

@ -1,7 +1,7 @@
var service = { mget: require('../service/mget') }; var service = { mget: require('../service/mget') };
var logger = require('pelias-logger').get('api'); var logger = require('pelias-logger').get('api');
function setup( config, backend ){ function setup( config, esclient ){
function controller( req, res, next ){ function controller( req, res, next ){
// do not run controller when a request // do not run controller when a request
@ -20,7 +20,7 @@ function setup( config, backend ){
logger.debug( '[ES req]', query ); logger.debug( '[ES req]', query );
service.mget( backend, query, function( err, docs ) { service.mget( esclient, query, function( err, docs ) {
// error handler // error handler
if( err ){ if( err ){
req.errors.push( err ); req.errors.push( err );

8
controller/search.js

@ -4,7 +4,7 @@ var service = { search: require('../service/search') };
var logger = require('pelias-logger').get('api'); var logger = require('pelias-logger').get('api');
var logging = require( '../helper/logging' ); var logging = require( '../helper/logging' );
function setup( config, backend, query ){ function setup( config, esclient, query ){
function controller( req, res, next ){ function controller( req, res, next ){
// do not run controller when a request // do not run controller when a request
// validation error has occurred. // validation error has occurred.
@ -33,7 +33,7 @@ function setup( config, backend, query ){
return next(); return next();
} }
// backend command // elasticsearch command
var cmd = { var cmd = {
index: config.indexName, index: config.indexName,
searchType: 'dfs_query_then_fetch', searchType: 'dfs_query_then_fetch',
@ -42,8 +42,8 @@ function setup( config, backend, query ){
logger.debug( '[ES req]', cmd ); logger.debug( '[ES req]', cmd );
// query backend // query elasticsearch
service.search( backend, cmd, function( err, docs, meta ){ service.search( esclient, cmd, function( err, docs, meta ){
// error handler // error handler
if( err ){ if( err ){

19
routes/v1.js

@ -1,6 +1,8 @@
var Router = require('express').Router; var Router = require('express').Router;
var reverseQuery = require('../query/reverse'); var reverseQuery = require('../query/reverse');
var backend = require('../src/backend');
const config = require( 'pelias-config' ).generate();
const esclient = require('elasticsearch').Client(config.esclient);
/** ----------------------- sanitizers ----------------------- **/ /** ----------------------- sanitizers ----------------------- **/
var sanitizers = { var sanitizers = {
@ -75,12 +77,11 @@ function addRoutes(app, peliasConfig) {
search: createRouter([ search: createRouter([
sanitizers.search.middleware, sanitizers.search.middleware,
middleware.calcSize(), middleware.calcSize(),
// 2nd parameter is `backend` which gets initialized internally
// 3rd parameter is which query module to use, use fallback/geodisambiguation // 3rd parameter is which query module to use, use fallback/geodisambiguation
// first, then use original search strategy if first query didn't return anything // first, then use original search strategy if first query didn't return anything
controllers.search(peliasConfig, backend, queries.libpostal), controllers.search(peliasConfig, esclient, queries.libpostal),
sanitizers.search_fallback.middleware, sanitizers.search_fallback.middleware,
controllers.search(peliasConfig, backend, queries.fallback_to_old_prod), controllers.search(peliasConfig, esclient, queries.fallback_to_old_prod),
postProc.trimByGranularity(), postProc.trimByGranularity(),
postProc.distances('focus.point.'), postProc.distances('focus.point.'),
postProc.confidenceScores(peliasConfig), postProc.confidenceScores(peliasConfig),
@ -98,7 +99,7 @@ function addRoutes(app, peliasConfig) {
structured: createRouter([ structured: createRouter([
sanitizers.structured_geocoding.middleware, sanitizers.structured_geocoding.middleware,
middleware.calcSize(), middleware.calcSize(),
controllers.search(peliasConfig, backend, queries.structured_geocoding), controllers.search(peliasConfig, esclient, queries.structured_geocoding),
postProc.trimByGranularityStructured(), postProc.trimByGranularityStructured(),
postProc.distances('focus.point.'), postProc.distances('focus.point.'),
postProc.confidenceScores(peliasConfig), postProc.confidenceScores(peliasConfig),
@ -115,7 +116,7 @@ function addRoutes(app, peliasConfig) {
]), ]),
autocomplete: createRouter([ autocomplete: createRouter([
sanitizers.autocomplete.middleware, sanitizers.autocomplete.middleware,
controllers.search(peliasConfig, backend, require('../query/autocomplete')), controllers.search(peliasConfig, esclient, require('../query/autocomplete')),
postProc.distances('focus.point.'), postProc.distances('focus.point.'),
postProc.confidenceScores(peliasConfig), postProc.confidenceScores(peliasConfig),
postProc.dedupe(), postProc.dedupe(),
@ -131,7 +132,7 @@ function addRoutes(app, peliasConfig) {
reverse: createRouter([ reverse: createRouter([
sanitizers.reverse.middleware, sanitizers.reverse.middleware,
middleware.calcSize(), middleware.calcSize(),
controllers.search(peliasConfig, backend, reverseQuery), controllers.search(peliasConfig, esclient, reverseQuery),
postProc.distances('point.'), postProc.distances('point.'),
// reverse confidence scoring depends on distance from origin // reverse confidence scoring depends on distance from origin
// so it must be calculated first // so it must be calculated first
@ -149,7 +150,7 @@ function addRoutes(app, peliasConfig) {
nearby: createRouter([ nearby: createRouter([
sanitizers.nearby.middleware, sanitizers.nearby.middleware,
middleware.calcSize(), middleware.calcSize(),
controllers.search(peliasConfig, backend, reverseQuery), controllers.search(peliasConfig, esclient, reverseQuery),
postProc.distances('point.'), postProc.distances('point.'),
// reverse confidence scoring depends on distance from origin // reverse confidence scoring depends on distance from origin
// so it must be calculated first // so it must be calculated first
@ -166,7 +167,7 @@ function addRoutes(app, peliasConfig) {
]), ]),
place: createRouter([ place: createRouter([
sanitizers.place.middleware, sanitizers.place.middleware,
controllers.place(peliasConfig, backend), controllers.place(peliasConfig, esclient),
postProc.accuracy(), postProc.accuracy(),
postProc.localNamingConventions(), postProc.localNamingConventions(),
postProc.renamePlacenames(), postProc.renamePlacenames(),

12
service/mget.js

@ -13,26 +13,26 @@
var logger = require( 'pelias-logger' ).get( 'api' ); var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, query, cb ){ function service( esclient, query, cb ){
// backend command // elasticsearch command
var cmd = { var cmd = {
body: { body: {
docs: query docs: query
} }
}; };
// query new backend // query elasticsearch
backend.client.mget( cmd, function( err, data ){ esclient.mget( cmd, function( err, data ){
// log total ms elasticsearch reported the query took to execute // log total ms elasticsearch reported the query took to execute
if( data && data.took ){ if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 ); logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
} }
// handle backend errors // handle elasticsearch errors
if( err ){ if( err ){
logger.error( `backend error ${err}`); logger.error( `elasticsearch error ${err}`);
return cb( err ); return cb( err );
} }

10
service/search.js

@ -7,19 +7,19 @@
var logger = require( 'pelias-logger' ).get( 'api' ); var logger = require( 'pelias-logger' ).get( 'api' );
function service( backend, cmd, cb ){ function service( esclient, cmd, cb ){
// query new backend // query elasticsearch
backend.client.search( cmd, function( err, data ){ esclient.search( cmd, function( err, data ){
// log total ms elasticsearch reported the query took to execute // log total ms elasticsearch reported the query took to execute
if( data && data.took ){ if( data && data.took ){
logger.verbose( 'time elasticsearch reported:', data.took / 1000 ); logger.verbose( 'time elasticsearch reported:', data.took / 1000 );
} }
// handle backend errors // handle elasticsearch errors
if( err ){ if( err ){
logger.error( `backend error ${err}` ); logger.error( `elasticsearch error ${err}` );
return cb( err ); return cb( err );
} }

6
src/backend.js

@ -1,6 +0,0 @@
const config = require( 'pelias-config' ).generate().esclient;
const client = require('elasticsearch').Client(config);
module.exports = {
client: client
};

2
test/unit/controller/place.js

@ -110,7 +110,7 @@ module.exports.tests.functional_failure = function(test, common) {
var controller = setup( fakeDefaultConfig, backend ); var controller = setup( fakeDefaultConfig, backend );
var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] }; var req = { clean: { ids: [ {'id' : 123, layers: [ 'b' ] } ] }, errors: [], warnings: [] };
var next = function( message ){ var next = function( message ){
t.equal(req.errors[0],'a backend error occurred','error passed to errorHandler'); t.equal(req.errors[0],'an elasticsearch error occurred','error passed to errorHandler');
t.end(); t.end();
}; };
controller(req, undefined, next ); controller(req, undefined, next );

2
test/unit/controller/search.js

@ -151,7 +151,7 @@ module.exports.tests.functional_failure = function(test, common) {
var controller = setup( fakeDefaultConfig, backend, mockQuery() ); var controller = setup( fakeDefaultConfig, backend, mockQuery() );
var req = { clean: { a: 'b' }, errors: [], warnings: [] }; var req = { clean: { a: 'b' }, errors: [], warnings: [] };
var next = function(){ var next = function(){
t.equal(req.errors[0],'a backend error occurred'); t.equal(req.errors[0],'an elasticsearch error occurred');
t.end(); t.end();
}; };
controller(req, undefined, next ); controller(req, undefined, next );

28
test/unit/mock/backend.js

@ -4,7 +4,7 @@ responses['client/suggest/ok/1'] = function( cmd, cb ){
return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' } ], [ { score: 2, text: 'mocktype:mockid2' } ]) ); return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' } ], [ { score: 2, text: 'mocktype:mockid2' } ]) );
}; };
responses['client/suggest/fail/1'] = function( cmd, cb ){ responses['client/suggest/fail/1'] = function( cmd, cb ){
return cb( 'a backend error occurred' ); return cb( 'an elasticsearch error occurred' );
}; };
responses['client/search/ok/1'] = function( cmd, cb ){ responses['client/search/ok/1'] = function( cmd, cb ){
return cb( undefined, searchEnvelope([{ return cb( undefined, searchEnvelope([{
@ -32,7 +32,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){
}])); }]));
}; };
responses['client/search/fail/1'] = function( cmd, cb ){ responses['client/search/fail/1'] = function( cmd, cb ){
return cb( 'a backend error occurred' ); return cb( 'an elasticsearch error occurred' );
}; };
responses['client/search/timeout/1'] = function( cmd, cb) { responses['client/search/timeout/1'] = function( cmd, cb) {
@ -73,19 +73,17 @@ responses['client/mget/fail/1'] = responses['client/search/fail/1'];
function setup( key, cmdCb ){ function setup( key, cmdCb ){
function backend( a, b ){ function backend( a, b ){
return { return {
client: { mget: function( cmd, cb ){
mget: function( cmd, cb ){ if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
if( 'function' === typeof cmdCb ){ cmdCb( cmd ); } return responses[key.indexOf('mget') === -1 ? 'client/mget/ok/1' : key].apply( this, arguments );
return responses[key.indexOf('mget') === -1 ? 'client/mget/ok/1' : key].apply( this, arguments ); },
}, suggest: function( cmd, cb ){
suggest: function( cmd, cb ){ if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
if( 'function' === typeof cmdCb ){ cmdCb( cmd ); } return responses[key].apply( this, arguments );
return responses[key].apply( this, arguments ); },
}, search: function( cmd, cb ){
search: function( cmd, cb ){ if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
if( 'function' === typeof cmdCb ){ cmdCb( cmd ); } return responses[key].apply( this, arguments );
return responses[key].apply( this, arguments );
}
} }
}; };
} }

5
test/unit/run.js

@ -11,8 +11,8 @@ var common = {
var tests = [ var tests = [
require('./app'), require('./app'),
require('./controller/index'), require('./controller/index'),
// require('./controller/place'), require('./controller/place'),
// require('./controller/search'), require('./controller/search'),
require('./helper/diffPlaces'), require('./helper/diffPlaces'),
require('./helper/geojsonify'), require('./helper/geojsonify'),
require('./helper/logging'), require('./helper/logging'),
@ -61,7 +61,6 @@ var tests = [
require('./sanitizer/_deprecate_quattroshapes'), require('./sanitizer/_deprecate_quattroshapes'),
require('./sanitizer/_categories'), require('./sanitizer/_categories'),
require('./sanitizer/nearby'), require('./sanitizer/nearby'),
require('./src/backend'),
require('./src/configValidation'), require('./src/configValidation'),
require('./sanitizer/autocomplete'), require('./sanitizer/autocomplete'),
require('./sanitizer/structured_geocoding'), require('./sanitizer/structured_geocoding'),

4
test/unit/service/mget.js

@ -79,7 +79,7 @@ module.exports.tests.functional_failure = function(test, common) {
get: () => { get: () => {
return { return {
error: (msg) => { error: (msg) => {
t.equal(msg, 'backend error a backend error occurred'); t.equal(msg, 'elasticsearch error an elasticsearch error occurred');
} }
}; };
} }
@ -88,7 +88,7 @@ module.exports.tests.functional_failure = function(test, common) {
}); });
service( backend, [ query ], function(err, data) { service( backend, [ query ], function(err, data) {
t.equal(err, 'a backend error occurred','error passed to errorHandler'); t.equal(err, 'an elasticsearch error occurred','error passed to errorHandler');
t.equal(data, undefined, 'data is undefined'); t.equal(data, undefined, 'data is undefined');
}); });
}); });

4
test/unit/service/search.js

@ -88,7 +88,7 @@ module.exports.tests.functional_failure = function(test, common) {
get: () => { get: () => {
return { return {
error: (msg) => { error: (msg) => {
t.equal(msg, 'backend error a backend error occurred'); t.equal(msg, 'elasticsearch error an elasticsearch error occurred');
} }
}; };
} }
@ -97,7 +97,7 @@ module.exports.tests.functional_failure = function(test, common) {
}); });
service( backend, [ query ], function(err, data) { service( backend, [ query ], function(err, data) {
t.equal(err, 'a backend error occurred','error passed to errorHandler'); t.equal(err, 'an elasticsearch error occurred','error passed to errorHandler');
t.equal(data, undefined, 'data is undefined'); t.equal(data, undefined, 'data is undefined');
}); });
}); });

40
test/unit/src/backend.js

@ -1,40 +0,0 @@
var proxyquire = require('proxyquire');
var stubConfig = {
generate: function generate() {
return {
esclient: {
hosts: [
'http://notLocalhost:9200',
'http://anotherHost:9200'
]
}
};
}
};
module.exports.tests = {};
module.exports.tests.config_properly_passed = function(test, common) {
test('Elasticsearch config is properly passed to elasticsearch module', function(t) {
var stubElasticsearchClient = {
Client: function(config) {
t.deepEquals(config.hosts, [ 'http://notLocalhost:9200', 'http://anotherHost:9200' ], 'hosts set correctly' );
t.end();
}
};
proxyquire('../../../src/backend', { 'pelias-config': stubConfig, 'elasticsearch': stubElasticsearchClient } );
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SANTIZE src/backend ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};
Loading…
Cancel
Save