diff --git a/app.js b/app.js index 21b465f0..e77bc3a2 100644 --- a/app.js +++ b/app.js @@ -22,7 +22,6 @@ var controllers = {}; controllers.index = require('./controller/index'); controllers.doc = require('./controller/doc'); controllers.suggest = require('./controller/suggest'); -controllers.suggest_nearby = require('./controller/suggest_nearby'); controllers.search = require('./controller/search'); /** ----------------------- routes ----------------------- **/ @@ -35,7 +34,9 @@ app.get( '/doc', sanitisers.doc.middleware, controllers.doc() ); // suggest API app.get( '/suggest', sanitisers.suggest.middleware, controllers.suggest() ); -app.get( '/suggest/nearby', sanitisers.suggest.middleware, controllers.suggest_nearby() ); +app.get( '/suggest/nearby', + sanitisers.suggest.middleware, + controllers.suggest(undefined, undefined, require('./helper/queryMixer').suggest_nearby) ); // search API app.get( '/search', sanitisers.search.middleware, controllers.search() ); diff --git a/controller/suggest.js b/controller/suggest.js index 914a8652..ef2774bc 100644 --- a/controller/suggest.js +++ b/controller/suggest.js @@ -4,46 +4,27 @@ var service = { mget: require('../service/mget') }; var geojsonify = require('../helper/geojsonify').search; -var async = require('async'); +var resultsHelper = require('../helper/results'); -function setup( backend, query ){ +function setup( backend, query, query_mixer ){ // allow overriding of dependencies backend = backend || require('../src/backend'); query = query || require('../query/suggest'); + query_mixer = query_mixer || require('../helper/queryMixer').suggest; function controller( req, res, next ){ + // backend command var cmd = { index: 'pelias', - body: query( req.clean ) + body: query( req.clean, query_mixer ) }; - var SIZE = req.clean.size || 10; + var size = req.clean.size || 10; - var query_backend = function(cmd, callback) { - // query backend - service.suggest( backend, cmd, function( err, docs ){ - - // error handler - if( err ){ return next( err ); } - - callback(null, docs); - }); - }; - - var dedup = function(combined) { - var unique_ids = []; - return combined.filter(function(item, pos) { - if (unique_ids.indexOf(item.text) === -1) { - unique_ids.push(item.text); - return true; - } - return false; - }); - }; - - var reply = function(docs) { + // responder + function reply( docs ){ // convert docs to geojson var geojson = geojsonify( docs ); @@ -53,17 +34,24 @@ function setup( backend, query ){ // respond return res.status(200).json( geojson ); - }; + } - var respond = function(data) { + // query backend + service.suggest( backend, cmd, function( err, suggested ){ + + // error handler + if( err ){ return next( err ); } + + // pick the required number of results + suggested = resultsHelper.picker(suggested, size); // no documents suggested, return empty array to avoid ActionRequestValidationException - if( !Array.isArray( data ) || !data.length ){ + if( !Array.isArray( suggested ) || !suggested.length ){ return reply([]); } // map suggester output to mget query - var query = data.map( function( doc ) { + var query = suggested.map( function( doc ) { var idParts = doc.text.split(':'); return { _index: 'pelias', @@ -81,72 +69,8 @@ function setup( backend, query ){ return reply( docs ); }); + }); - }; - - if (req.clean.input) { - var async_query; - - // admin only - req.admin = {}; - for (var k in req.clean) { req.admin[k] = req.clean[k]; } - req.admin.layers = ['admin0','admin1','admin2']; - - if (req.clean.input.length < 4 && isNaN(parseInt(req.clean.input, 10))) { - async_query = { - admin_3p: function(callback){ - cmd.body = query( req.admin, 3 ); - query_backend(cmd, callback); - }, - admin_1p: function(callback){ - cmd.body = query( req.admin, 1 ); - query_backend(cmd, callback); - }, - all_3p: function(callback) { - cmd.body = query( req.clean, 3 ); - query_backend(cmd, callback); - } - }; - } else { - async_query = { - all_5p: function(callback){ - cmd.body = query( req.clean, 5); - query_backend(cmd, callback); - }, - all_3p: function(callback){ - cmd.body = query( req.clean, 3); - query_backend(cmd, callback); - }, - all_1p: function(callback){ - cmd.body = query( req.clean, 1 ); - query_backend(cmd, callback); - }, - admin_1p: function(callback){ - cmd.body = query( req.admin ); - query_backend(cmd, callback); - } - }; - } - - async.parallel(async_query, function(err, results) { - // results is equal to: {a: docs, b: docs, c: docs} - var splice_length = parseInt((SIZE / Object.keys(results).length), 10); - var results_keys = Object.keys(async_query); - - var combined = []; - results_keys.forEach(function(key){ - combined = combined.concat(results[key].splice(0,splice_length)); - }); - - combined = dedup(combined); - respond(combined); - }); - } else { - query_backend(cmd, function(err, results) { - respond(results); - }); - } - } return controller; diff --git a/controller/suggest_nearby.js b/controller/suggest_nearby.js deleted file mode 100644 index 6016cb7f..00000000 --- a/controller/suggest_nearby.js +++ /dev/null @@ -1,72 +0,0 @@ - -var service = { - suggest: require('../service/suggest'), - mget: require('../service/mget') -}; -var geojsonify = require('../helper/geojsonify').search; - -function setup( backend, query ){ - - // allow overriding of dependencies - backend = backend || require('../src/backend'); - query = query || require('../query/suggest'); - - function controller( req, res, next ){ - - // backend command - var cmd = { - index: 'pelias', - body: query( req.clean ) - }; - - // responder - function reply( docs ){ - - // convert docs to geojson - var geojson = geojsonify( docs ); - - // response envelope - geojson.date = new Date().getTime(); - - // respond - return res.status(200).json( geojson ); - } - - // query backend - service.suggest( backend, cmd, function( err, suggested ){ - - // error handler - if( err ){ return next( err ); } - - // no documents suggested, return empty array to avoid ActionRequestValidationException - if( !Array.isArray( suggested ) || !suggested.length ){ - return reply([]); - } - - // map suggester output to mget query - var query = suggested.map( function( doc ) { - var idParts = doc.text.split(':'); - return { - _index: 'pelias', - _type: idParts[0], - _id: idParts.slice(1).join(':') - }; - }); - - service.mget( backend, query, function( err, docs ){ - - // error handler - if( err ){ return next( err ); } - - // reply - return reply( docs ); - - }); - }); - - } - - return controller; -} - -module.exports = setup; \ No newline at end of file diff --git a/helper/queryMixer.json b/helper/queryMixer.json new file mode 100644 index 00000000..e3becfba --- /dev/null +++ b/helper/queryMixer.json @@ -0,0 +1,28 @@ +{ + "suggest": [ + { + "layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"], + "precision": [5, 3, 1] + }, + { + "layers": ["admin0","admin1","admin2","neighborhood"], + "precision": [] + }, + { + "layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"], + "precision": [3], + "fuzzy": "AUTO" + } + ], + "suggest_nearby": [ + { + "layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"], + "precision": [] + }, + { + "layers": ["geoname","osmnode","osmway","admin0","admin1","admin2","neighborhood"], + "precision": [], + "fuzzy": "AUTO" + } + ] +} \ No newline at end of file diff --git a/helper/results.js b/helper/results.js new file mode 100644 index 00000000..1d0ec1fd --- /dev/null +++ b/helper/results.js @@ -0,0 +1,48 @@ + +var picker = function( results, size ){ + var combined = []; + var num_results = 0; + + for (var i=0; i 0) ? sort_by_score(combined) : combined; +}; + +var dedup = function(arr) { + var unique_ids = []; + return arr.filter(function(item, pos) { + if (unique_ids.indexOf(item.name.default) === -1) { + unique_ids.push(item.name.default); + return true; + } + return false; + }); +}; + +var sort_by_score = function(arr) { + return arr.map(function(doc) { + return doc.sort(function(a,b) { + return b.score - a.score; + }); + }).reduce(function(a,b) { //flatten + return a.concat(b); + }); +}; + +module.exports = { + picker: picker, + dedup: dedup +}; \ No newline at end of file diff --git a/query/suggest.js b/query/suggest.js index 29f2f0b3..4e9d30b6 100644 --- a/query/suggest.js +++ b/query/suggest.js @@ -2,9 +2,17 @@ var logger = require('../src/logger'); // Build pelias suggest query -function generate( params, precision ){ +function generate( params, query_mixer, fuzziness ){ - var getPrecision = function(zoom) { + var CmdGenerator = function(params){ + this.params = params; + this.cmd = { + 'text': params.input + }; + }; + + CmdGenerator.prototype.get_precision = function() { + var zoom = this.params.zoom; switch (true) { case (zoom > 15): return 5; // zoom: >= 16 @@ -16,28 +24,46 @@ function generate( params, precision ){ return 2; // zoom: 4-5 default: return 1; // zoom: 1-3 or when zoom: undefined - } + } }; - var cmd = { - 'pelias' : { - 'text' : params.input, + CmdGenerator.prototype.add_suggester = function(name, precision, layers, fuzzy) { + this.cmd[name] = { 'completion' : { - 'size' : params.size, + 'size' : this.params.size, 'field' : 'suggest', 'context': { - 'dataset': params.layers, + 'dataset': layers || this.params.layers, 'location': { - 'value': [ params.lon, params.lat ], - 'precision': precision || getPrecision(params.zoom) + 'value': [ this.params.lon, this.params.lat ], + 'precision': precision || this.get_precision() } + }, + 'fuzzy': { + 'fuzziness': fuzzy || fuzziness || 0 } } - } + }; }; - // logger.log( 'cmd', JSON.stringify( cmd, null, 2 ) ); - return cmd; + var cmd = new CmdGenerator(params); + if (query_mixer && query_mixer.length) { + query_mixer.forEach(function(item, index){ + if (item.precision && Array.isArray( item.precision ) && item.precision.length ) { + item.precision.forEach(function(precision) { + cmd.add_suggester(index, precision, item.layers, item.fuzzy); + }); + } else { + cmd.add_suggester(index, undefined, item.layers, item.fuzzy); + } + }); + } else { + cmd.add_suggester(0); + } + + + // logger.log( 'cmd', JSON.stringify( cmd.cmd, null, 2 ) ); + return cmd.cmd; } diff --git a/service/suggest.js b/service/suggest.js index fdf82fe9..f27b67c9 100644 --- a/service/suggest.js +++ b/service/suggest.js @@ -6,21 +6,36 @@ **/ function service( backend, cmd, cb ){ - // query new backend backend().client.suggest( cmd, function( err, data ){ - // handle backend errors if( err ){ return cb( err ); } - + // map returned documents + var docs = []; - if( data && Array.isArray( data.pelias ) && data.pelias.length ){ - docs = data.pelias[0].options || []; + var unique_ids = []; + var num_keys = Object.keys(data).length; + var has_docs = function(obj) { + return Array.isArray( obj ) && obj.length && obj[0].options && obj[0].options.length; + }; + for (var i=0, j=0; i 0, true, 'valid key'); + t.equal(Array.isArray( mix ), true, 'is an array'); + t.equal(mix.length > 0, true, 'is not an empty array'); + mix.forEach( function(this_mix) { + t.notEqual(Object.getOwnPropertyNames(this_mix).length, 0, 'object not empty'); + for (var keys in this_mix) { + t.notEqual(valid_keys.indexOf(keys), -1, keys + ' is valid'); + switch(keys) { + case 'fuzzy': + t.notEqual(valid_fuzzy_vals.indexOf(this_mix[keys]), -1, 'fuzzy value ' + this_mix[keys] + ' is valid'); + break; + case 'layers': + t.equal(Array.isArray(this_mix[keys]), true, 'layers is an array'); + t.equal(this_mix[keys].length > 0, true, 'layers is not an empty array'); + isValidLayer(t, this_mix[keys]); + break; + case 'precision': + t.equal(Array.isArray( this_mix[keys] ), true, keys + ' is an array'); + if (this_mix[keys].length > 0) { + isValidPrecision(t, this_mix[keys]); + } + break; + default: + break; + } + } + }); + t.end(); + }); + }; + + for (var keys in query_mixer) { + isValid(keys, query_mixer[keys]); + } +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('query_mixer: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; \ No newline at end of file diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js index 597b139c..fe617fe7 100644 --- a/test/unit/mock/backend.js +++ b/test/unit/mock/backend.js @@ -1,7 +1,7 @@ var responses = {}; responses['client/suggest/ok/1'] = function( cmd, cb ){ - return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' }, { score: 2, text: 'mocktype:mockid2' } ]) ); + return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' } ], [ { score: 2, text: 'mocktype:mockid2' } ]) ); }; responses['client/suggest/fail/1'] = function( cmd, cb ){ return cb( 'a backend error occurred' ); @@ -82,8 +82,8 @@ function mgetEnvelope( options ){ return { docs: options }; } -function suggestEnvelope( options ){ - return { pelias: [{ options: options }]}; +function suggestEnvelope( options1, options2 ){ + return { 0: [{ options: options1 }], 1: [{ options: options2 }]}; } function searchEnvelope( options ){ diff --git a/test/unit/query/suggest.js b/test/unit/query/suggest.js index 77c3f099..96d86fdd 100644 --- a/test/unit/query/suggest.js +++ b/test/unit/query/suggest.js @@ -18,8 +18,8 @@ module.exports.tests.query = function(test, common) { layers: ['test'] }); var expected = { - pelias: { - text: 'test', + text: 'test', + 0: { completion: { field: 'suggest', size: 10, @@ -29,7 +29,8 @@ module.exports.tests.query = function(test, common) { precision: 1, value: [ 0, 0 ] } - } + }, + fuzzy: { fuzziness: 0 }, } } }; @@ -71,8 +72,8 @@ module.exports.tests.precision = function(test, common) { layers: ['test'] }); var expected = { - pelias: { - text: 'test', + text: 'test', + 0: { completion: { field: 'suggest', size: 10, @@ -82,7 +83,8 @@ module.exports.tests.precision = function(test, common) { precision: test_case.precision, value: [ 0, 0 ] } - } + }, + fuzzy: { fuzziness: 0 }, } } }; @@ -92,6 +94,38 @@ module.exports.tests.precision = function(test, common) { }); }; +module.exports.tests.fuzziness = function(test, common) { + var test_cases = [0,1,2,'AUTO', undefined, null, '']; + test('valid fuzziness', function(t) { + test_cases.forEach( function( test_case ){ + var query = generate({ + input: 'test', size: 10, + lat: 0, lon: 0, zoom:0, + layers: ['test'] + }, undefined, test_case); + var expected = { + text: 'test', + 0: { + completion: { + field: 'suggest', + size: 10, + context: { + dataset: [ 'test' ], + location: { + precision: 1, + value: [ 0, 0 ] + } + }, + fuzzy: { fuzziness: test_case || 0 }, + } + } + }; + t.deepEqual(query, expected, 'valid suggest query for fuziness = ' + test_case); + }); + t.end(); + }); +}; + module.exports.all = function (tape, common) { function test(name, testFunction) { diff --git a/test/unit/run.js b/test/unit/run.js index 0bc64a72..ee45800b 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -6,7 +6,6 @@ var tests = [ require('./controller/index'), require('./controller/doc'), require('./controller/suggest'), - require('./controller/suggest_nearby'), require('./controller/search'), require('./service/mget'), require('./service/search'), @@ -18,7 +17,8 @@ var tests = [ require('./query/search'), require('./query/reverse'), require('./helper/geojsonify'), - require('./helper/outputSchema') + require('./helper/outputSchema'), + require('./helper/queryMixer') ]; tests.map(function(t) { diff --git a/test/unit/service/suggest.js b/test/unit/service/suggest.js index a058b57e..08471e09 100644 --- a/test/unit/service/suggest.js +++ b/test/unit/service/suggest.js @@ -17,8 +17,8 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.functional_success = function(test, common) { var expected = [ - { score: 1, text: 'mocktype:mockid1' }, - { score: 2, text: 'mocktype:mockid2' } + [{ score: 1, text: 'mocktype:mockid1' }], + [{ score: 2, text: 'mocktype:mockid2' }] ]; test('valid ES query', function(t) {