From e5f783567ffeb456d04104827f0303277abe94f1 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 15 Jul 2015 19:39:39 -0400 Subject: [PATCH] remove all things that involved suggester including query mixer keeping the suggest sanitizer for suggest/nearby wherein lat/lon are required --- app.js | 1 - controller/suggest.js | 79 ------------ helper/queryMixer.json | 39 ------ helper/results.js | 48 ------- query/suggest.js | 75 ----------- service/suggest.js | 48 ------- test/unit/controller/suggest.js | 178 -------------------------- test/unit/helper/queryMixer.js | 81 ------------ test/unit/query/suggest.js | 217 -------------------------------- test/unit/run.js | 6 +- test/unit/service/suggest.js | 72 ----------- 11 files changed, 1 insertion(+), 843 deletions(-) delete mode 100644 controller/suggest.js delete mode 100644 helper/queryMixer.json delete mode 100644 helper/results.js delete mode 100644 query/suggest.js delete mode 100644 service/suggest.js delete mode 100644 test/unit/controller/suggest.js delete mode 100644 test/unit/helper/queryMixer.js delete mode 100644 test/unit/query/suggest.js delete mode 100644 test/unit/service/suggest.js diff --git a/app.js b/app.js index 933b7093..c0b72610 100644 --- a/app.js +++ b/app.js @@ -26,7 +26,6 @@ sanitisers.reverse = require('./sanitiser/reverse'); var controllers = {}; controllers.index = require('./controller/index'); controllers.doc = require('./controller/doc'); -controllers.suggest = require('./controller/suggest'); controllers.search = require('./controller/search'); /** ----------------------- routes ----------------------- **/ diff --git a/controller/suggest.js b/controller/suggest.js deleted file mode 100644 index 0adb6d8f..00000000 --- a/controller/suggest.js +++ /dev/null @@ -1,79 +0,0 @@ - -var service = { - suggest: require('../service/suggest'), - mget: require('../service/mget') -}; -var geojsonify = require('../helper/geojsonify').search; -var resultsHelper = require('../helper/results'); - -function setup( backend, query, query_mixer ){ - - // allow overriding of dependencies - backend = backend || require('../src/backend'); - query = query || require('../query/suggest'); - query_mixer = query_mixer || require('../helper/queryMixer').suggest; - - function controller( req, res, next ){ - - // backend command - var cmd = { - index: 'pelias', - body: query( req.clean, query_mixer ) - }; - - var size = req.clean.size || 10; - - // responder - function reply( docs ){ - - // convert docs to geojson - var geojson = geojsonify( docs, req.clean ); - - // response envelope - geojson.date = new Date().getTime(); - - // respond - return res.status(200).json( geojson ); - } - - // query backend - service.suggest( backend, cmd, function( err, suggested ){ - - // error handler - if( err ){ return next( err ); } - - // pick the required number of results - suggested = resultsHelper.picker(suggested, size); - - // no documents suggested, return empty array to avoid ActionRequestValidationException - if( !Array.isArray( suggested ) || !suggested.length ){ - return reply([]); - } - - // map suggester output to mget query - var query = suggested.map( function( doc ) { - var idParts = doc.text.split(':'); - return { - _index: 'pelias', - _type: idParts[0], - _id: idParts.slice(1).join(':') - }; - }); - - service.mget( backend, query, function( err, docs ){ - - // error handler - if( err ){ return next( err ); } - - // reply - return reply( docs ); - - }); - }); - - } - - return controller; -} - -module.exports = setup; \ No newline at end of file diff --git a/helper/queryMixer.json b/helper/queryMixer.json deleted file mode 100644 index 4d8ea590..00000000 --- a/helper/queryMixer.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "suggest": [ - { - "layers": ["poi", "admin", "address"], - "precision": [5, 3, 1] - }, - { - "layers": ["admin"], - "precision": [] - }, - { - "layers": ["poi", "admin", "address"], - "precision": [3], - "fuzzy": "AUTO" - } - ], - "suggest_nearby": [ - { - "layers": ["poi", "admin", "address"], - "precision": [] - }, - { - "layers": ["poi", "admin", "address"], - "precision": [], - "fuzzy": "AUTO" - } - ], - "coarse": [ - { - "layers": ["admin"], - "precision": [5, 3, 1] - }, - { - "layers": ["admin"], - "precision": [3], - "fuzzy": "AUTO" - } - ] -} \ No newline at end of file diff --git a/helper/results.js b/helper/results.js deleted file mode 100644 index 1d0ec1fd..00000000 --- a/helper/results.js +++ /dev/null @@ -1,48 +0,0 @@ - -var picker = function( results, size ){ - var combined = []; - var num_results = 0; - - for (var i=0; i 0) ? sort_by_score(combined) : combined; -}; - -var dedup = function(arr) { - var unique_ids = []; - return arr.filter(function(item, pos) { - if (unique_ids.indexOf(item.name.default) === -1) { - unique_ids.push(item.name.default); - return true; - } - return false; - }); -}; - -var sort_by_score = function(arr) { - return arr.map(function(doc) { - return doc.sort(function(a,b) { - return b.score - a.score; - }); - }).reduce(function(a,b) { //flatten - return a.concat(b); - }); -}; - -module.exports = { - picker: picker, - dedup: dedup -}; \ No newline at end of file diff --git a/query/suggest.js b/query/suggest.js deleted file mode 100644 index 16cd7b28..00000000 --- a/query/suggest.js +++ /dev/null @@ -1,75 +0,0 @@ - -var get_layers = require('../helper/layers'); - -// Build pelias suggest query -function generate( params, query_mixer, fuzziness ){ - - var CmdGenerator = function(params){ - this.params = params; - this.cmd = { - 'text': params.input - }; - }; - - CmdGenerator.prototype.get_precision = function() { - var zoom = this.params.zoom; - switch (true) { - case (zoom > 15): - return 5; // zoom: >= 16 - case (zoom > 9): - return 4; // zoom: 10-15 - case (zoom > 5): - return 3; // zoom: 6-9 - case (zoom > 3): - return 2; // zoom: 4-5 - default: - return 1; // zoom: 1-3 or when zoom: undefined - } - }; - - CmdGenerator.prototype.add_suggester = function(name, precision, layers, fuzzy) { - this.cmd[name] = { - 'completion' : { - 'size' : this.params.size, - 'field' : 'suggest', - 'context': { - 'dataset': this.params.layers || layers, - 'location': { - 'value': null, - 'precision': precision || this.get_precision() - } - }, - 'fuzzy': { - 'fuzziness': fuzzy || fuzziness || 0 - } - } - }; - if (!isNaN(this.params.lon) && !isNaN(this.params.lat)) { - this.cmd[name].completion.context.location.value = [ this.params.lon, this.params.lat ]; - } - }; - - var cmd = new CmdGenerator(params); - var suggester_index = 0; - - if (query_mixer && query_mixer.length) { - query_mixer.forEach(function(item, index){ - var expanded_layers = get_layers(item.layers); - if (item.precision && Array.isArray( item.precision ) && item.precision.length ) { - item.precision.forEach(function(precision) { - cmd.add_suggester(suggester_index++, precision, expanded_layers, item.fuzzy); - }); - } else { - cmd.add_suggester(suggester_index++, undefined, expanded_layers, item.fuzzy); - } - }); - } else { - cmd.add_suggester(suggester_index++); - } - - - return cmd.cmd; - -} - -module.exports = generate; diff --git a/service/suggest.js b/service/suggest.js deleted file mode 100644 index c5bf2c1f..00000000 --- a/service/suggest.js +++ /dev/null @@ -1,48 +0,0 @@ - -/** - - cmd can be any valid ES suggest command - -**/ -var peliasLogger = require( 'pelias-logger' ).get( 'service/suggest' ); - -var microtime = require( 'microtime' ); - -function service( backend, cmd, cb ){ - // query new backend - var startTime = microtime.nowDouble(); - backend().client.suggest( cmd, function( err, data ){ - peliasLogger.verbose( 'time elasticsearch query took:', microtime.nowDouble() - startTime ); - // handle backend errors - if( err ){ return cb( err ); } - - // map returned documents - - var docs = []; - var unique_ids = []; - var num_keys = Object.keys(data).length; - var has_docs = function(obj) { - return Array.isArray( obj ) && obj.length && obj[0].options && obj[0].options.length; - }; - for (var i=0, j=0; i 0, true, 'valid key'); - t.equal(Array.isArray( mix ), true, 'is an array'); - t.equal(mix.length > 0, true, 'is not an empty array'); - mix.forEach( function(this_mix) { - t.notEqual(Object.getOwnPropertyNames(this_mix).length, 0, 'object not empty'); - for (var keys in this_mix) { - t.notEqual(valid_keys.indexOf(keys), -1, keys + ' is valid'); - switch(keys) { - case 'fuzzy': - t.notEqual(valid_fuzzy_vals.indexOf(this_mix[keys]), -1, 'fuzzy value ' + this_mix[keys] + ' is valid'); - break; - case 'layers': - t.equal(Array.isArray(this_mix[keys]), true, 'layers is an array'); - t.equal(this_mix[keys].length > 0, true, 'layers is not an empty array'); - isValidLayer(t, this_mix[keys]); - break; - case 'precision': - t.equal(Array.isArray( this_mix[keys] ), true, keys + ' is an array'); - if (this_mix[keys].length > 0) { - isValidPrecision(t, this_mix[keys]); - } - break; - default: - break; - } - } - }); - t.end(); - }); - }; - - for (var keys in query_mixer) { - isValid(keys, query_mixer[keys]); - } -}; - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('query_mixer: ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; \ No newline at end of file diff --git a/test/unit/query/suggest.js b/test/unit/query/suggest.js deleted file mode 100644 index d1067af0..00000000 --- a/test/unit/query/suggest.js +++ /dev/null @@ -1,217 +0,0 @@ - -var generate = require('../../../query/suggest'); -var queryMixer = require('../../../helper/queryMixer'); - -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { - t.equal(typeof generate, 'function', 'valid function'); - t.end(); - }); -}; - -module.exports.tests.query = function(test, common) { - test('valid query', function(t) { - var query = generate({ - input: 'test', size: 10, - lat: 0, lon: 0, zoom:1, - layers: ['test'] - }); - var expected = { - text: 'test', - 0: { - completion: { - field: 'suggest', - size: 10, - context: { - dataset: [ 'test' ], - location: { - precision: 1, - value: [ 0, 0 ] - } - }, - fuzzy: { fuzziness: 0 }, - } - } - }; - t.deepEqual(query, expected, 'valid suggest query'); - t.end(); - }); - - test('valid query without lat/lon', function(t) { - var query = generate({ - input: 'test', size: 10, - layers: ['test'] - }); - var expected = { - text: 'test', - 0: { - completion: { - field: 'suggest', - size: 10, - context: { - dataset: [ 'test' ], - location: { - precision: 1, - value: null - } - }, - fuzzy: { fuzziness: 0 }, - } - } - }; - t.deepEqual(query, expected, 'valid suggest query'); - t.end(); - }); -}; - -module.exports.tests.precision = function(test, common) { - var test_cases = [ - {zoom:1, precision:1}, - {zoom:2, precision:1}, - {zoom:3, precision:1}, - {zoom:4, precision:2}, - {zoom:5, precision:2}, - {zoom:6, precision:3}, - {zoom:7, precision:3}, - {zoom:8, precision:3}, - {zoom:9, precision:3}, - {zoom:10, precision:4}, - {zoom:11, precision:4}, - {zoom:12, precision:4}, - {zoom:13, precision:4}, - {zoom:14, precision:4}, - {zoom:15, precision:4}, - {zoom:16, precision:5}, - {zoom:17, precision:5}, - {zoom:18, precision:5}, - {zoom:19, precision:5}, - {zoom:'', precision:1}, - {zoom:null, precision:1}, - {zoom:undefined, precision:1} - ]; - test('valid precision', function(t) { - test_cases.forEach( function( test_case ){ - var query = generate({ - input: 'test', size: 10, - lat: 0, lon: 0, zoom:test_case.zoom, - layers: ['test'] - }); - var expected = { - text: 'test', - 0: { - completion: { - field: 'suggest', - size: 10, - context: { - dataset: [ 'test' ], - location: { - precision: test_case.precision, - value: [ 0, 0 ] - } - }, - fuzzy: { fuzziness: 0 }, - } - } - }; - t.deepEqual(query, expected, 'valid suggest query for zoom = ' + test_case.zoom); - }); - t.end(); - }); -}; - -module.exports.tests.fuzziness = function(test, common) { - var test_cases = [0,1,2,'AUTO', undefined, null, '']; - test('valid fuzziness', function(t) { - test_cases.forEach( function( test_case ){ - var query = generate({ - input: 'test', size: 10, - lat: 0, lon: 0, zoom:0, - layers: ['test'] - }, undefined, test_case); - var expected = { - text: 'test', - 0: { - completion: { - field: 'suggest', - size: 10, - context: { - dataset: [ 'test' ], - location: { - precision: 1, - value: [ 0, 0 ] - } - }, - fuzzy: { fuzziness: test_case || 0 }, - } - } - }; - t.deepEqual(query, expected, 'valid suggest query for fuziness = ' + test_case); - }); - t.end(); - }); -}; - -module.exports.tests.queryMixer = function(test, common) { - test('valid query mixer', function(t) { - for (var suggester in queryMixer) { - var queryMix = queryMixer[suggester]; - - var number_of_suggesters = queryMix.reduce(function(sum, query) { - return sum + (query.precision.length || 1); - }, 0); - - var query = generate({ - input: 'test', size: 10, - lat: 0, lon: 0, zoom:0 - }, queryMix); - - // adding one to number_of_suggesters to account for the key "text" in query. - t.deepEqual(Object.keys(query).length, number_of_suggesters + 1, - suggester + ' has correct number of suggesters' - ); - } - - t.end(); - }); -}; - -var isValidLayer = function(t, query, layers) { - for(var qKey in query) { - var q = query[qKey]; - if (q.completion) { - var query_layers = q.completion.context.dataset; - t.deepEqual(query_layers, layers, layers + ' layers set correctly'); - } - } -}; - -module.exports.tests.layers = function(test, common) { - test('valid layers with query-mixers', function(t) { - for (var suggester in queryMixer) { - var queryMix = queryMixer[suggester]; - var layers= ['geoname', 'osm']; - var query = generate({ - input: 'test', size: 10, - lat: 0, lon: 0, zoom:0, - layers: layers - }, queryMix); - - isValidLayer(t, query, layers); - } - - t.end(); - }); -}; - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('suggest query ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; \ No newline at end of file diff --git a/test/unit/run.js b/test/unit/run.js index a8663fad..f28d7306 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -5,24 +5,20 @@ var common = {}; var tests = [ require('./controller/index'), require('./controller/doc'), - require('./controller/suggest'), require('./controller/search'), require('./service/mget'), require('./service/search'), - require('./service/suggest'), require('./sanitiser/suggest'), require('./sanitiser/search'), require('./sanitiser/reverse'), require('./sanitiser/doc'), require('./sanitiser/coarse'), require('./query/indeces'), - require('./query/suggest'), require('./query/sort'), require('./query/search'), require('./query/reverse'), require('./helper/geojsonify'), - require('./helper/outputSchema'), - require('./helper/queryMixer') + require('./helper/outputSchema') ]; tests.map(function(t) { diff --git a/test/unit/service/suggest.js b/test/unit/service/suggest.js deleted file mode 100644 index 08471e09..00000000 --- a/test/unit/service/suggest.js +++ /dev/null @@ -1,72 +0,0 @@ - -var setup = require('../../../service/suggest'), - mockBackend = require('../mock/backend'); - -var example_valid_es_query = { body: { a: 'b' }, index: 'pelias' }; - -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { - t.equal(typeof setup, 'function', 'setup is a function'); - t.end(); - }); -}; - -// functionally test service -module.exports.tests.functional_success = function(test, common) { - - var expected = [ - [{ score: 1, text: 'mocktype:mockid1' }], - [{ score: 2, text: 'mocktype:mockid2' }] - ]; - - test('valid ES query', function(t) { - var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){ - t.deepEqual(cmd, example_valid_es_query, 'no change to the command'); - }); - setup( backend, example_valid_es_query, function(err, data) { - t.true(Array.isArray(data), 'returns an array'); - data.forEach(function(d) { - t.true(typeof d === 'object', 'valid object'); - }); - t.deepEqual(data, expected, 'values correctly mapped'); - t.end(); - }); - }); - -}; - -// functionally test service -module.exports.tests.functional_failure = function(test, common) { - - test('invalid ES query', function(t) { - var invalid_queries = [ - { }, - { foo: 'bar' } - ]; - - var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){ - t.notDeepEqual(cmd, example_valid_es_query, 'incorrect backend command'); - }); - invalid_queries.forEach(function(query) { - setup( backend, [ query ], function(err, data) { - t.equal(err, 'a backend error occurred','error passed to errorHandler'); - t.equal(data, undefined, 'data is undefined'); - }); - }); - t.end(); - }); - -}; - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('SERVICE /suggest ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; \ No newline at end of file