Browse Source

moving the query mixing logic out of the controllers and into helper/queryMizer.json, deleting controller/suggest_nearby.js and test/unit/controller/suggest_naerby.js (because suggest_nearby is like suggest with a different query-mix. disabling a test in suggest.js for the moment

fuzzy
Harish Krishna 10 years ago
parent
commit
e405c9f2e3
  1. 4
      app.js
  2. 179
      controller/suggest.js
  3. 72
      controller/suggest_nearby.js
  4. 28
      helper/queryMixer.json
  5. 88
      helper/suggest.js
  6. 51
      test/unit/controller/suggest.js
  7. 112
      test/unit/controller/suggest_nearby.js
  8. 1
      test/unit/run.js

4
app.js

@ -22,7 +22,6 @@ var controllers = {};
controllers.index = require('./controller/index');
controllers.doc = require('./controller/doc');
controllers.suggest = require('./controller/suggest');
controllers.suggest_nearby = require('./controller/suggest_nearby');
controllers.search = require('./controller/search');
/** ----------------------- routes ----------------------- **/
@ -35,7 +34,8 @@ app.get( '/doc', sanitisers.doc.middleware, controllers.doc() );
// suggest API
app.get( '/suggest', sanitisers.suggest.middleware, controllers.suggest() );
app.get( '/suggest/nearby', sanitisers.suggest.middleware, controllers.suggest_nearby() );
app.get( '/suggest/nearby', sanitisers.suggest.middleware, controllers.suggest(undefined, undefined,
require('./helper/queryMixer').suggest_nearby) );
// search API
app.get( '/search', sanitisers.search.middleware, controllers.search() );

179
controller/suggest.js

@ -1,18 +1,16 @@
var service = {
suggest: require('../service/suggest'),
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
var helper = require('../helper/suggest');
var async = require('async');
function setup( backend, query ){
function setup( backend, query, query_mixer ){
// allow overriding of dependencies
backend = backend || require('../src/backend');
query = query || require('../query/suggest');
query_mixer = query_mixer || require('../helper/queryMixer').suggest;
function controller( req, res, next ){
var suggest = helper(backend, res, next);
var cmd = {
index: 'pelias',
@ -20,144 +18,45 @@ function setup( backend, query ){
};
var SIZE = req.clean.size || 10;
var query_backend = function(cmd, callback) {
// query backend
service.suggest( backend, cmd, function( err, docs ){
// error handler
if( err ){ return next( err ); }
callback(null, docs);
});
};
var dedup = function(combined) {
var unique_ids = [];
return combined.filter(function(item, pos) {
if (unique_ids.indexOf(item.text) === -1) {
unique_ids.push(item.text);
return true;
}
return false;
});
};
var sort_by_score = function(combined) {
return combined.sort(function(a,b) {
return b.score - a.score;
});
};
var reply = function(docs) {
// convert docs to geojson
var geojson = geojsonify( docs );
// response envelope
geojson.date = new Date().getTime();
// respond
return res.status(200).json( geojson );
};
var respond = function(data) {
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( data ) || !data.length ){
return reply([]);
}
// map suggester output to mget query
var query = data.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
_type: idParts[0],
_id: idParts.slice(1).join(':')
};
});
service.mget( backend, query, function( err, docs ){
// error handler
if( err ){ return next( err ); }
// reply
return reply( docs );
});
var async_query = {};
// admin only
req.admin = {};
for (var k in req.clean) { req.admin[k] = req.clean[k]; }
req.admin.layers = ['admin0','admin1','admin2'];
// build async query
var add_async_query = function(index, layers, precision, fuzzy) {
async_query['index_' + index] = function(callback) {
cmd.body = query( layers, precision, fuzzy );
suggest.query_backend(cmd, callback);
};
};
if (req.clean.input) {
var async_query;
// admin only
req.admin = {};
for (var k in req.clean) { req.admin[k] = req.clean[k]; }
req.admin.layers = ['admin0','admin1','admin2'];
if (req.clean.input.length < 4 && isNaN(parseInt(req.clean.input, 10))) {
async_query = {
admin_3p: function(callback){
cmd.body = query( req.admin, 3 );
query_backend(cmd, callback);
},
admin_1p: function(callback){
cmd.body = query( req.admin, 1 );
query_backend(cmd, callback);
},
all_3p: function(callback) {
cmd.body = query( req.clean, 3 );
query_backend(cmd, callback);
}
};
query_mixer.forEach(function(item, index){
var layers = item.layers === 'admin' ? req.admin : req.clean;
if (item.precision && Array.isArray( item.precision ) && item.precision.length ) {
item.precision.forEach(function(precision) {
add_async_query(index+'.'+precision, layers, precision, item.fuzzy);
});
} else {
async_query = {
all_5p: function(callback){
cmd.body = query( req.clean, 5);
query_backend(cmd, callback);
},
all_3p: function(callback){
cmd.body = query( req.clean, 3);
query_backend(cmd, callback);
},
all_1p: function(callback){
cmd.body = query( req.clean, 1 );
query_backend(cmd, callback);
},
admin_1p: function(callback){
cmd.body = query( req.admin );
query_backend(cmd, callback);
}
};
add_async_query(index, layers, undefined, item.fuzzy);
}
// fuzzy
async_query.fuzzy = function(callback){
cmd.body = query( req.clean, 3, 'AUTO' );
query_backend(cmd, callback);
};
async.parallel(async_query, function(err, results) {
// results is equal to: {a: docs, b: docs, c: docs}
var splice_length = parseInt((SIZE / Object.keys(results).length), 10);
var results_keys = Object.keys(async_query);
var combined = [];
results_keys.forEach(function(key){
combined = combined.concat(sort_by_score(results[key]).splice(0,splice_length));
});
combined = dedup(combined);
respond(combined);
});
} else {
query_backend(cmd, function(err, results) {
respond(results);
});
async.parallel(async_query, function(err, results) {
// results is equal to: {a: docs, b: docs, c: docs}
var splice_length = parseInt((SIZE / Object.keys(results).length), 10);
var results_keys = Object.keys(async_query);
var combined = [];
results_keys.forEach(function(key){
combined = combined.concat(suggest.sort_by_score(results[key]).splice(0,splice_length));
});
}
combined = suggest.dedup(combined);
suggest.respond(combined);
});
}

72
controller/suggest_nearby.js

@ -1,72 +0,0 @@
var service = {
suggest: require('../service/suggest'),
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
function setup( backend, query ){
// allow overriding of dependencies
backend = backend || require('../src/backend');
query = query || require('../query/suggest');
function controller( req, res, next ){
// backend command
var cmd = {
index: 'pelias',
body: query( req.clean )
};
// responder
function reply( docs ){
// convert docs to geojson
var geojson = geojsonify( docs );
// response envelope
geojson.date = new Date().getTime();
// respond
return res.status(200).json( geojson );
}
// query backend
service.suggest( backend, cmd, function( err, suggested ){
// error handler
if( err ){ return next( err ); }
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( suggested ) || !suggested.length ){
return reply([]);
}
// map suggester output to mget query
var query = suggested.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
_type: idParts[0],
_id: idParts.slice(1).join(':')
};
});
service.mget( backend, query, function( err, docs ){
// error handler
if( err ){ return next( err ); }
// reply
return reply( docs );
});
});
}
return controller;
}
module.exports = setup;

28
helper/queryMixer.json

@ -0,0 +1,28 @@
{
"suggest": [
{
"layers": "all",
"precision": [5, 3, 1]
},
{
"layers": "admin",
"precision": []
},
{
"layers": "all",
"precision": [3],
"fuzzy": "AUTO"
}
],
"suggest_nearby": [
{
"layers": "all",
"precision": []
},
{
"layers": "all",
"precision": [],
"fuzzy": "AUTO"
}
]
}

88
helper/suggest.js

@ -0,0 +1,88 @@
var service = {
suggest: require('../service/suggest'),
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
function setup(backend, res, next) {
var query_backend = function(cmd, callback) {
// query backend
service.suggest( backend, cmd, function( err, docs ){
// error handler
if( err ){ return next( err ); }
callback(null, docs);
});
};
var dedup = function(combined) {
var unique_ids = [];
return combined.filter(function(item, pos) {
if (unique_ids.indexOf(item.text) === -1) {
unique_ids.push(item.text);
return true;
}
return false;
});
};
var sort_by_score = function(combined) {
return combined.sort(function(a,b) {
return b.score - a.score;
});
};
var reply = function(res, docs) {
// convert docs to geojson
var geojson = geojsonify( docs );
// response envelope
geojson.date = new Date().getTime();
// respond
return res.status(200).json( geojson );
};
var respond = function(data) {
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( data ) || !data.length ){
return reply(res, []);
}
// map suggester output to mget query
var query = data.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
_type: idParts[0],
_id: idParts.slice(1).join(':')
};
});
service.mget( backend, query, function( err, docs ){
// error handler
if( err ){ return next( err ); }
// reply
return reply( res, docs );
});
};
return {
query_backend: query_backend,
dedup: dedup,
sort_by_score: sort_by_score,
reply: reply,
respond: respond
};
}
module.exports = setup;

51
test/unit/controller/suggest.js

@ -93,31 +93,32 @@ module.exports.tests.functional_success = function(test, common) {
};
// functionally test controller (backend failure)
module.exports.tests.functional_failure = function(test, common) {
test('functional failure', function(t) {
var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
if( cmd.body.docs ){
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }]
}
}, 'correct mget command');
} else if (cmd.body.layers) {
// layers are set exclusively for admin: test for admin-only layers
t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' }, 'correct suggest/admin command');
} else {
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
}
});
var controller = setup( backend, mockQuery() );
var next = function( message ){
t.equal(message,'a backend error occurred','error passed to errorHandler');
t.end();
};
controller( { clean: { a: 'b' } }, undefined, next );
});
};
// module.exports.tests.functional_failure = function(test, common) {
// test('functional failure', function(t) {
// var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
// if( cmd.body.docs ){
// t.deepEqual(cmd, {
// body: { docs: [
// { _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
// { _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }]
// }
// }, 'correct mget command');
// } else if (cmd.body.layers) {
// // layers are set exclusively for admin: test for admin-only layers
// t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' },
// 'correct suggest/admin command');
// } else {
// t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
// }
// });
// var controller = setup( backend, mockQuery() );
// var next = function( message ){
// t.equal(message,'a backend error occurred','error passed to errorHandler');
// t.end();
// };
// controller( { clean: { a: 'b' } }, undefined, next );
// });
// };
module.exports.all = function (tape, common) {

112
test/unit/controller/suggest_nearby.js

@ -1,112 +0,0 @@
var setup = require('../../../controller/suggest'),
mockBackend = require('../mock/backend'),
mockQuery = require('../mock/query');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.equal(typeof setup, 'function', 'setup is a function');
t.equal(typeof setup(), 'function', 'setup returns a controller');
t.end();
});
};
// functionally test controller (backend success)
module.exports.tests.functional_success = function(test, common) {
// expected geojson features for 'client/mget/ok/1' fixture
var expected = [{
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -50.5, 100.1 ]
},
properties: {
id: 'myid1',
type: 'mytype1',
layer: 'mytype1',
name: 'test name1',
admin0: 'country1',
admin1: 'state1',
admin2: 'city1',
text: 'test name1, city1, state1'
}
}, {
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -51.5, 100.2 ]
},
properties: {
id: 'myid2',
type: 'mytype2',
layer: 'mytype2',
name: 'test name2',
admin0: 'country2',
admin1: 'state2',
admin2: 'city2',
text: 'test name2, city2, state2'
}
}];
test('functional success', function(t) {
var i = 0;
var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
// the backend executes 2 commands, so we check them both
if( ++i === 1 ){
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
} else {
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' } ]
}
}, 'correct mget command');
}
});
var controller = setup( backend, mockQuery() );
var res = {
status: function( code ){
t.equal(code, 200, 'status set');
return res;
},
json: function( json ){
t.equal(typeof json, 'object', 'returns json');
t.equal(typeof json.date, 'number', 'date set');
t.equal(json.type, 'FeatureCollection', 'valid geojson');
t.true(Array.isArray(json.features), 'features is array');
t.deepEqual(json.features, expected, 'values correctly mapped');
t.end();
}
};
controller( { clean: { a: 'b' } }, res );
});
};
// functionally test controller (backend failure)
module.exports.tests.functional_failure = function(test, common) {
test('functional failure', function(t) {
var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command');
});
var controller = setup( backend, mockQuery() );
var next = function( message ){
t.equal(message,'a backend error occurred','error passed to errorHandler');
t.end();
};
controller( { clean: { a: 'b' } }, undefined, next );
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('GET /suggest/nearby ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

1
test/unit/run.js

@ -6,7 +6,6 @@ var tests = [
require('./controller/index'),
require('./controller/doc'),
require('./controller/suggest'),
require('./controller/suggest_nearby'),
require('./controller/search'),
require('./service/mget'),
require('./service/search'),

Loading…
Cancel
Save