Browse Source

Merge pull request #59 from pelias/one-query-multiple-suggesters

One query multiple suggesters
pull/64/head
Harish Krishna 10 years ago
parent
commit
6a3c5d32ca
  1. 5
      app.js
  2. 116
      controller/suggest.js
  3. 72
      controller/suggest_nearby.js
  4. 28
      helper/queryMixer.json
  5. 48
      helper/results.js
  6. 52
      query/suggest.js
  7. 27
      service/suggest.js
  8. 13
      test/unit/controller/suggest.js
  9. 112
      test/unit/controller/suggest_nearby.js
  10. 80
      test/unit/helper/queryMixer.js
  11. 6
      test/unit/mock/backend.js
  12. 46
      test/unit/query/suggest.js
  13. 4
      test/unit/run.js
  14. 4
      test/unit/service/suggest.js

5
app.js

@ -22,7 +22,6 @@ var controllers = {};
controllers.index = require('./controller/index');
controllers.doc = require('./controller/doc');
controllers.suggest = require('./controller/suggest');
controllers.suggest_nearby = require('./controller/suggest_nearby');
controllers.search = require('./controller/search');
/** ----------------------- routes ----------------------- **/
@ -35,7 +34,9 @@ app.get( '/doc', sanitisers.doc.middleware, controllers.doc() );
// suggest API
app.get( '/suggest', sanitisers.suggest.middleware, controllers.suggest() );
app.get( '/suggest/nearby', sanitisers.suggest.middleware, controllers.suggest_nearby() );
app.get( '/suggest/nearby',
sanitisers.suggest.middleware,
controllers.suggest(undefined, undefined, require('./helper/queryMixer').suggest_nearby) );
// search API
app.get( '/search', sanitisers.search.middleware, controllers.search() );

116
controller/suggest.js

@ -4,46 +4,27 @@ var service = {
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
var async = require('async');
var resultsHelper = require('../helper/results');
function setup( backend, query ){
function setup( backend, query, query_mixer ){
// allow overriding of dependencies
backend = backend || require('../src/backend');
query = query || require('../query/suggest');
query_mixer = query_mixer || require('../helper/queryMixer').suggest;
function controller( req, res, next ){
// backend command
var cmd = {
index: 'pelias',
body: query( req.clean )
body: query( req.clean, query_mixer )
};
var SIZE = req.clean.size || 10;
var size = req.clean.size || 10;
var query_backend = function(cmd, callback) {
// query backend
service.suggest( backend, cmd, function( err, docs ){
// error handler
if( err ){ return next( err ); }
callback(null, docs);
});
};
var dedup = function(combined) {
var unique_ids = [];
return combined.filter(function(item, pos) {
if (unique_ids.indexOf(item.text) === -1) {
unique_ids.push(item.text);
return true;
}
return false;
});
};
var reply = function(docs) {
// responder
function reply( docs ){
// convert docs to geojson
var geojson = geojsonify( docs );
@ -53,17 +34,24 @@ function setup( backend, query ){
// respond
return res.status(200).json( geojson );
};
}
var respond = function(data) {
// query backend
service.suggest( backend, cmd, function( err, suggested ){
// error handler
if( err ){ return next( err ); }
// pick the required number of results
suggested = resultsHelper.picker(suggested, size);
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( data ) || !data.length ){
if( !Array.isArray( suggested ) || !suggested.length ){
return reply([]);
}
// map suggester output to mget query
var query = data.map( function( doc ) {
var query = suggested.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
@ -81,72 +69,8 @@ function setup( backend, query ){
return reply( docs );
});
});
};
if (req.clean.input) {
var async_query;
// admin only
req.admin = {};
for (var k in req.clean) { req.admin[k] = req.clean[k]; }
req.admin.layers = ['admin0','admin1','admin2'];
if (req.clean.input.length < 4 && isNaN(parseInt(req.clean.input, 10))) {
async_query = {
admin_3p: function(callback){
cmd.body = query( req.admin, 3 );
query_backend(cmd, callback);
},
admin_1p: function(callback){
cmd.body = query( req.admin, 1 );
query_backend(cmd, callback);
},
all_3p: function(callback) {
cmd.body = query( req.clean, 3 );
query_backend(cmd, callback);
}
};
} else {
async_query = {
all_5p: function(callback){
cmd.body = query( req.clean, 5);
query_backend(cmd, callback);
},
all_3p: function(callback){
cmd.body = query( req.clean, 3);
query_backend(cmd, callback);
},
all_1p: function(callback){
cmd.body = query( req.clean, 1 );
query_backend(cmd, callback);
},
admin_1p: function(callback){
cmd.body = query( req.admin );
query_backend(cmd, callback);
}
};
}
async.parallel(async_query, function(err, results) {
// results is equal to: {a: docs, b: docs, c: docs}
var splice_length = parseInt((SIZE / Object.keys(results).length), 10);
var results_keys = Object.keys(async_query);
var combined = [];
results_keys.forEach(function(key){
combined = combined.concat(results[key].splice(0,splice_length));
});
combined = dedup(combined);
respond(combined);
});
} else {
query_backend(cmd, function(err, results) {
respond(results);
});
}
}
return controller;

72
controller/suggest_nearby.js

@ -1,72 +0,0 @@
var service = {
suggest: require('../service/suggest'),
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
function setup( backend, query ){
// allow overriding of dependencies
backend = backend || require('../src/backend');
query = query || require('../query/suggest');
function controller( req, res, next ){
// backend command
var cmd = {
index: 'pelias',
body: query( req.clean )
};
// responder
function reply( docs ){
// convert docs to geojson
var geojson = geojsonify( docs );
// response envelope
geojson.date = new Date().getTime();
// respond
return res.status(200).json( geojson );
}
// query backend
service.suggest( backend, cmd, function( err, suggested ){
// error handler
if( err ){ return next( err ); }
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( suggested ) || !suggested.length ){
return reply([]);
}
// map suggester output to mget query
var query = suggested.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
_type: idParts[0],
_id: idParts.slice(1).join(':')
};
});
service.mget( backend, query, function( err, docs ){
// error handler
if( err ){ return next( err ); }
// reply
return reply( docs );
});
});
}
return controller;
}
module.exports = setup;

28
helper/queryMixer.json

@ -0,0 +1,28 @@
{
"suggest": [
{
"layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"],
"precision": [5, 3, 1]
},
{
"layers": ["admin0","admin1","admin2","neighborhood"],
"precision": []
},
{
"layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"],
"precision": [3],
"fuzzy": "AUTO"
}
],
"suggest_nearby": [
{
"layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"],
"precision": []
},
{
"layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"],
"precision": [],
"fuzzy": "AUTO"
}
]
}

48
helper/results.js

@ -0,0 +1,48 @@
var picker = function( results, size ){
var combined = [];
var num_results = 0;
for (var i=0; i<results.length && num_results<size; i++) {
if (results[i] && results[i].length) {
combined[i] = combined[i] || [];
combined[i].push(results[i][0]);
results[i].splice(0,1);
num_results++;
} else {
results.splice(i,1);
i--;
}
if (i === results.length-1) {
i=0;
}
}
return (combined.length > 0) ? sort_by_score(combined) : combined;
};
var dedup = function(arr) {
var unique_ids = [];
return arr.filter(function(item, pos) {
if (unique_ids.indexOf(item.name.default) === -1) {
unique_ids.push(item.name.default);
return true;
}
return false;
});
};
var sort_by_score = function(arr) {
return arr.map(function(doc) {
return doc.sort(function(a,b) {
return b.score - a.score;
});
}).reduce(function(a,b) { //flatten
return a.concat(b);
});
};
module.exports = {
picker: picker,
dedup: dedup
};

52
query/suggest.js

@ -2,9 +2,17 @@
var logger = require('../src/logger');
// Build pelias suggest query
function generate( params, precision ){
function generate( params, query_mixer, fuzziness ){
var getPrecision = function(zoom) {
var CmdGenerator = function(params){
this.params = params;
this.cmd = {
'text': params.input
};
};
CmdGenerator.prototype.get_precision = function() {
var zoom = this.params.zoom;
switch (true) {
case (zoom > 15):
return 5; // zoom: >= 16
@ -16,28 +24,46 @@ function generate( params, precision ){
return 2; // zoom: 4-5
default:
return 1; // zoom: 1-3 or when zoom: undefined
}
}
};
var cmd = {
'pelias' : {
'text' : params.input,
CmdGenerator.prototype.add_suggester = function(name, precision, layers, fuzzy) {
this.cmd[name] = {
'completion' : {
'size' : params.size,
'size' : this.params.size,
'field' : 'suggest',
'context': {
'dataset': params.layers,
'dataset': layers || this.params.layers,
'location': {
'value': [ params.lon, params.lat ],
'precision': precision || getPrecision(params.zoom)
'value': [ this.params.lon, this.params.lat ],
'precision': precision || this.get_precision()
}
},
'fuzzy': {
'fuzziness': fuzzy || fuzziness || 0
}
}
}
};
};
// logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) );
return cmd;
var cmd = new CmdGenerator(params);
if (query_mixer && query_mixer.length) {
query_mixer.forEach(function(item, index){
if (item.precision && Array.isArray( item.precision ) && item.precision.length ) {
item.precision.forEach(function(precision) {
cmd.add_suggester(index, precision, item.layers, item.fuzzy);
});
} else {
cmd.add_suggester(index, undefined, item.layers, item.fuzzy);
}
});
} else {
cmd.add_suggester(0);
}
// logger.log( 'cmd', JSON.stringify( cmd.cmd, null, 2 ) );
return cmd.cmd;
}

27
service/suggest.js

@ -6,21 +6,36 @@
**/
function service( backend, cmd, cb ){
// query new backend
backend().client.suggest( cmd, function( err, data ){
// handle backend errors
if( err ){ return cb( err ); }
// map returned documents
var docs = [];
if( data && Array.isArray( data.pelias ) && data.pelias.length ){
docs = data.pelias[0].options || [];
var unique_ids = [];
var num_keys = Object.keys(data).length;
var has_docs = function(obj) {
return Array.isArray( obj ) && obj.length && obj[0].options && obj[0].options.length;
};
for (var i=0, j=0; i<num_keys && j<num_keys; i++) {
if ( has_docs(data[i]) ){
docs[i] = docs[i] || [];
var res = data[i][0].options[0];
if (unique_ids.indexOf(res.text) === -1) {
docs[i].push(res);
unique_ids.push(res.text);
}
data[i][0].options.splice(0,1);
} else {
j++;
}
i = i === num_keys-1 ? 1 : i;
}
// fire callback
return cb( null, docs );
return cb( null, docs);
});
}

13
test/unit/controller/suggest.js

@ -61,14 +61,6 @@ module.exports.tests.functional_success = function(test, common) {
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }
]}
}, 'correct mget command');
} else if (cmd.body.layers) {
// layers are set exclusively for admin: test for admin-only layers
t.deepEqual(cmd, {
body: {
input: 'b',
layers: [ 'admin0', 'admin1', 'admin2' ]
}, index: 'pelias'
}, 'correct suggest/admin command');
} else {
t.deepEqual(cmd, { body: { input: 'b' }, index: 'pelias' }, 'correct suggest command');
}
@ -105,7 +97,8 @@ module.exports.tests.functional_failure = function(test, common) {
}, 'correct mget command');
} else if (cmd.body.layers) {
// layers are set exclusively for admin: test for admin-only layers
t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' }, 'correct suggest/admin command');
t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' },
'correct suggest/admin command');
} else {
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
}
@ -113,9 +106,9 @@ module.exports.tests.functional_failure = function(test, common) {
var controller = setup( backend, mockQuery() );
var next = function( message ){
t.equal(message,'a backend error occurred','error passed to errorHandler');
t.end();
};
controller( { clean: { a: 'b' } }, undefined, next );
t.end();
});
};

112
test/unit/controller/suggest_nearby.js

@ -1,112 +0,0 @@
var setup = require('../../../controller/suggest'),
mockBackend = require('../mock/backend'),
mockQuery = require('../mock/query');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.equal(typeof setup, 'function', 'setup is a function');
t.equal(typeof setup(), 'function', 'setup returns a controller');
t.end();
});
};
// functionally test controller (backend success)
module.exports.tests.functional_success = function(test, common) {
// expected geojson features for 'client/mget/ok/1' fixture
var expected = [{
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -50.5, 100.1 ]
},
properties: {
id: 'myid1',
type: 'mytype1',
layer: 'mytype1',
name: 'test name1',
admin0: 'country1',
admin1: 'state1',
admin2: 'city1',
text: 'test name1, city1, state1'
}
}, {
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -51.5, 100.2 ]
},
properties: {
id: 'myid2',
type: 'mytype2',
layer: 'mytype2',
name: 'test name2',
admin0: 'country2',
admin1: 'state2',
admin2: 'city2',
text: 'test name2, city2, state2'
}
}];
test('functional success', function(t) {
var i = 0;
var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
// the backend executes 2 commands, so we check them both
if( ++i === 1 ){
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
} else {
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' } ]
}
}, 'correct mget command');
}
});
var controller = setup( backend, mockQuery() );
var res = {
status: function( code ){
t.equal(code, 200, 'status set');
return res;
},
json: function( json ){
t.equal(typeof json, 'object', 'returns json');
t.equal(typeof json.date, 'number', 'date set');
t.equal(json.type, 'FeatureCollection', 'valid geojson');
t.true(Array.isArray(json.features), 'features is array');
t.deepEqual(json.features, expected, 'values correctly mapped');
t.end();
}
};
controller( { clean: { a: 'b' } }, res );
});
};
// functionally test controller (backend failure)
module.exports.tests.functional_failure = function(test, common) {
test('functional failure', function(t) {
var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command');
});
var controller = setup( backend, mockQuery() );
var next = function( message ){
t.equal(message,'a backend error occurred','error passed to errorHandler');
t.end();
};
controller( { clean: { a: 'b' } }, undefined, next );
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('GET /suggest/nearby ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

80
test/unit/helper/queryMixer.js

@ -0,0 +1,80 @@
var query_mixer = require('../../../helper/queryMixer.json');
var indeces = require('../../../query/indeces');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('interface', function(t) {
t.equal(typeof query_mixer, 'object', 'valid object');
t.equal(query_mixer.hasOwnProperty('suggest'), true, 'has suggest defined');
t.equal(query_mixer.hasOwnProperty('suggest_nearby'), true, 'has suggest_nearby defined');
t.end();
});
};
module.exports.tests.valid = function(test, common) {
var valid_keys = ['layers', 'precision', 'fuzzy'];
var valid_fuzzy_vals = ['AUTO', 0, 1, 2];
var valid_layer_vals = indeces;
var isValidPrecision = function(t, precisionArr) {
precisionArr.forEach(function(precision) {
t.notEqual(isNaN(precision), true, precision + ' is a valid precision value');
});
};
var isValidLayer = function(t, layerArr) {
layerArr.forEach(function(this_layer) {
t.notEqual(valid_layer_vals.indexOf(this_layer), -1, 'layer value ' + this_layer + ' is valid');
});
};
var isValid = function(key, mix) {
test('valid mix (' + key + ')' , function(t) {
t.equal(keys.length > 0, true, 'valid key');
t.equal(Array.isArray( mix ), true, 'is an array');
t.equal(mix.length > 0, true, 'is not an empty array');
mix.forEach( function(this_mix) {
t.notEqual(Object.getOwnPropertyNames(this_mix).length, 0, 'object not empty');
for (var keys in this_mix) {
t.notEqual(valid_keys.indexOf(keys), -1, keys + ' is valid');
switch(keys) {
case 'fuzzy':
t.notEqual(valid_fuzzy_vals.indexOf(this_mix[keys]), -1, 'fuzzy value ' + this_mix[keys] + ' is valid');
break;
case 'layers':
t.equal(Array.isArray(this_mix[keys]), true, 'layers is an array');
t.equal(this_mix[keys].length > 0, true, 'layers is not an empty array');
isValidLayer(t, this_mix[keys]);
break;
case 'precision':
t.equal(Array.isArray( this_mix[keys] ), true, keys + ' is an array');
if (this_mix[keys].length > 0) {
isValidPrecision(t, this_mix[keys]);
}
break;
default:
break;
}
}
});
t.end();
});
};
for (var keys in query_mixer) {
isValid(keys, query_mixer[keys]);
}
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('query_mixer: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

6
test/unit/mock/backend.js

@ -1,7 +1,7 @@
var responses = {};
responses['client/suggest/ok/1'] = function( cmd, cb ){
return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' }, { score: 2, text: 'mocktype:mockid2' } ]) );
return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' } ], [ { score: 2, text: 'mocktype:mockid2' } ]) );
};
responses['client/suggest/fail/1'] = function( cmd, cb ){
return cb( 'a backend error occurred' );
@ -82,8 +82,8 @@ function mgetEnvelope( options ){
return { docs: options };
}
function suggestEnvelope( options ){
return { pelias: [{ options: options }]};
function suggestEnvelope( options1, options2 ){
return { 0: [{ options: options1 }], 1: [{ options: options2 }]};
}
function searchEnvelope( options ){

46
test/unit/query/suggest.js

@ -18,8 +18,8 @@ module.exports.tests.query = function(test, common) {
layers: ['test']
});
var expected = {
pelias: {
text: 'test',
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
@ -29,7 +29,8 @@ module.exports.tests.query = function(test, common) {
precision: 1,
value: [ 0, 0 ]
}
}
},
fuzzy: { fuzziness: 0 },
}
}
};
@ -71,8 +72,8 @@ module.exports.tests.precision = function(test, common) {
layers: ['test']
});
var expected = {
pelias: {
text: 'test',
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
@ -82,7 +83,8 @@ module.exports.tests.precision = function(test, common) {
precision: test_case.precision,
value: [ 0, 0 ]
}
}
},
fuzzy: { fuzziness: 0 },
}
}
};
@ -92,6 +94,38 @@ module.exports.tests.precision = function(test, common) {
});
};
module.exports.tests.fuzziness = function(test, common) {
var test_cases = [0,1,2,'AUTO', undefined, null, ''];
test('valid fuzziness', function(t) {
test_cases.forEach( function( test_case ){
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:0,
layers: ['test']
}, undefined, test_case);
var expected = {
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
context: {
dataset: [ 'test' ],
location: {
precision: 1,
value: [ 0, 0 ]
}
},
fuzzy: { fuzziness: test_case || 0 },
}
}
};
t.deepEqual(query, expected, 'valid suggest query for fuziness = ' + test_case);
});
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {

4
test/unit/run.js

@ -6,7 +6,6 @@ var tests = [
require('./controller/index'),
require('./controller/doc'),
require('./controller/suggest'),
require('./controller/suggest_nearby'),
require('./controller/search'),
require('./service/mget'),
require('./service/search'),
@ -18,7 +17,8 @@ var tests = [
require('./query/search'),
require('./query/reverse'),
require('./helper/geojsonify'),
require('./helper/outputSchema')
require('./helper/outputSchema'),
require('./helper/queryMixer')
];
tests.map(function(t) {

4
test/unit/service/suggest.js

@ -17,8 +17,8 @@ module.exports.tests.interface = function(test, common) {
module.exports.tests.functional_success = function(test, common) {
var expected = [
{ score: 1, text: 'mocktype:mockid1' },
{ score: 2, text: 'mocktype:mockid2' }
[{ score: 1, text: 'mocktype:mockid1' }],
[{ score: 2, text: 'mocktype:mockid2' }]
];
test('valid ES query', function(t) {

Loading…
Cancel
Save