pelias
/
api
mirror of https://github.com/pelias/api.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

remove all things that involved suggester including query mixer 

keeping the suggest sanitizer for suggest/nearby wherein lat/lon are required
pull/165/head
Harish Krishna 10 years ago
parent
68dbf1bd2e
commit
e5f783567f
11 changed files with 1 additions and 843 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      app.js
  2. 79
      controller/suggest.js
  3. 39
      helper/queryMixer.json
  4. 48
      helper/results.js
  5. 75
      query/suggest.js
  6. 48
      service/suggest.js
  7. 178
      test/unit/controller/suggest.js
  8. 81
      test/unit/helper/queryMixer.js
  9. 217
      test/unit/query/suggest.js
  10. 6
      test/unit/run.js
  11. 72
      test/unit/service/suggest.js

1
app.js
Unescape View File

@ -26,7 +26,6 @@ sanitisers.reverse = require('./sanitiser/reverse');
var controllers = {};
controllers.index = require('./controller/index');
controllers.doc = require('./controller/doc');
controllers.suggest = require('./controller/suggest');
controllers.search = require('./controller/search');
/** ----------------------- routes ----------------------- **/

79
controller/suggest.js
Unescape View File

@ -1,79 +0,0 @@
var service = {
suggest: require('../service/suggest'),
mget: require('../service/mget')
};
var geojsonify = require('../helper/geojsonify').search;
var resultsHelper = require('../helper/results');
function setup( backend, query, query_mixer ){
// allow overriding of dependencies
backend = backend || require('../src/backend');
query = query || require('../query/suggest');
query_mixer = query_mixer || require('../helper/queryMixer').suggest;
function controller( req, res, next ){
// backend command
var cmd = {
index: 'pelias',
body: query( req.clean, query_mixer )
};
var size = req.clean.size || 10;
// responder
function reply( docs ){
// convert docs to geojson
var geojson = geojsonify( docs, req.clean );
// response envelope
geojson.date = new Date().getTime();
// respond
return res.status(200).json( geojson );
}
// query backend
service.suggest( backend, cmd, function( err, suggested ){
// error handler
if( err ){ return next( err ); }
// pick the required number of results
suggested = resultsHelper.picker(suggested, size);
// no documents suggested, return empty array to avoid ActionRequestValidationException
if( !Array.isArray( suggested ) || !suggested.length ){
return reply([]);
}
// map suggester output to mget query
var query = suggested.map( function( doc ) {
var idParts = doc.text.split(':');
return {
_index: 'pelias',
_type: idParts[0],
_id: idParts.slice(1).join(':')
};
});
service.mget( backend, query, function( err, docs ){
// error handler
if( err ){ return next( err ); }
// reply
return reply( docs );
});
});
}
return controller;
}
module.exports = setup;

39
helper/queryMixer.json
Unescape View File

@ -1,39 +0,0 @@
{
"suggest": [
{
"layers": ["poi", "admin", "address"],
"precision": [5, 3, 1]
},
{
"layers": ["admin"],
"precision": []
},
{
"layers": ["poi", "admin", "address"],
"precision": [3],
"fuzzy": "AUTO"
}
],
"suggest_nearby": [
{
"layers": ["poi", "admin", "address"],
"precision": []
},
{
"layers": ["poi", "admin", "address"],
"precision": [],
"fuzzy": "AUTO"
}
],
"coarse": [
{
"layers": ["admin"],
"precision": [5, 3, 1]
},
{
"layers": ["admin"],
"precision": [3],
"fuzzy": "AUTO"
}
]
}

48
helper/results.js
Unescape View File

@ -1,48 +0,0 @@
var picker = function( results, size ){
var combined = [];
var num_results = 0;
for (var i=0; i<results.length && num_results<size; i++) {
if (results[i] && results[i].length) {
combined[i] = combined[i] || [];
combined[i].push(results[i][0]);
results[i].splice(0,1);
num_results++;
} else {
results.splice(i,1);
i--;
}
if (i === results.length-1) {
i=0;
}
}
return (combined.length > 0) ? sort_by_score(combined) : combined;
};
var dedup = function(arr) {
var unique_ids = [];
return arr.filter(function(item, pos) {
if (unique_ids.indexOf(item.name.default) === -1) {
unique_ids.push(item.name.default);
return true;
}
return false;
});
};
var sort_by_score = function(arr) {
return arr.map(function(doc) {
return doc.sort(function(a,b) {
return b.score - a.score;
});
}).reduce(function(a,b) { //flatten
return a.concat(b);
});
};
module.exports = {
picker: picker,
dedup: dedup
};

75
query/suggest.js
Unescape View File

@ -1,75 +0,0 @@
var get_layers = require('../helper/layers');
// Build pelias suggest query
function generate( params, query_mixer, fuzziness ){
var CmdGenerator = function(params){
this.params = params;
this.cmd = {
'text': params.input
};
};
CmdGenerator.prototype.get_precision = function() {
var zoom = this.params.zoom;
switch (true) {
case (zoom > 15):
return 5; // zoom: >= 16
case (zoom > 9):
return 4; // zoom: 10-15
case (zoom > 5):
return 3; // zoom: 6-9
case (zoom > 3):
return 2; // zoom: 4-5
default:
return 1; // zoom: 1-3 or when zoom: undefined
}
};
CmdGenerator.prototype.add_suggester = function(name, precision, layers, fuzzy) {
this.cmd[name] = {
'completion' : {
'size' : this.params.size,
'field' : 'suggest',
'context': {
'dataset': this.params.layers || layers,
'location': {
'value': null,
'precision': precision || this.get_precision()
}
},
'fuzzy': {
'fuzziness': fuzzy || fuzziness || 0
}
}
};
if (!isNaN(this.params.lon) && !isNaN(this.params.lat)) {
this.cmd[name].completion.context.location.value = [ this.params.lon, this.params.lat ];
}
};
var cmd = new CmdGenerator(params);
var suggester_index = 0;
if (query_mixer && query_mixer.length) {
query_mixer.forEach(function(item, index){
var expanded_layers = get_layers(item.layers);
if (item.precision && Array.isArray( item.precision ) && item.precision.length ) {
item.precision.forEach(function(precision) {
cmd.add_suggester(suggester_index++, precision, expanded_layers, item.fuzzy);
});
} else {
cmd.add_suggester(suggester_index++, undefined, expanded_layers, item.fuzzy);
}
});
} else {
cmd.add_suggester(suggester_index++);
}
return cmd.cmd;
}
module.exports = generate;

48
service/suggest.js
Unescape View File

@ -1,48 +0,0 @@
/**
cmd can be any valid ES suggest command
**/
var peliasLogger = require( 'pelias-logger' ).get( 'service/suggest' );
var microtime = require( 'microtime' );
function service( backend, cmd, cb ){
// query new backend
var startTime = microtime.nowDouble();
backend().client.suggest( cmd, function( err, data ){
peliasLogger.verbose( 'time elasticsearch query took:', microtime.nowDouble() - startTime );
// handle backend errors
if( err ){ return cb( err ); }
// map returned documents
var docs = [];
var unique_ids = [];
var num_keys = Object.keys(data).length;
var has_docs = function(obj) {
return Array.isArray( obj ) && obj.length && obj[0].options && obj[0].options.length;
};
for (var i=0, j=0; i<num_keys && j<num_keys; i++) {
if ( has_docs(data[i]) ){
docs[i] = docs[i] || [];
var res = data[i][0].options[0];
if (unique_ids.indexOf(res.text) === -1) {
docs[i].push(res);
unique_ids.push(res.text);
}
data[i][0].options.splice(0,1);
} else {
j++;
}
i = i === num_keys-1 ? 1 : i;
}
// fire callback
return cb( null, docs);
});
}
module.exports = service;

178
test/unit/controller/suggest.js
Unescape View File

@ -1,178 +0,0 @@
var setup = require('../../../controller/suggest'),
mockBackend = require('../mock/backend'),
mockQuery = require('../mock/query');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.equal(typeof setup, 'function', 'setup is a function');
t.equal(typeof setup(), 'function', 'setup returns a controller');
t.end();
});
};
// functionally test controller (backend success)
module.exports.tests.functional_success = function(test, common) {
// expected geojson features for 'client/mget/ok/1' fixture
var expected = [{
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -50.5, 100.1 ]
},
properties: {
id: 'myid1',
layer: 'mytype1',
text: 'test name1, city1, state1'
}
}, {
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -51.5, 100.2 ]
},
properties: {
id: 'myid2',
layer: 'mytype2',
text: 'test name2, city2, state2'
}
}];
test('functional success', function(t) {
var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
// the backend executes suggest (vanilla and admin-only) and mget, so we check them all based on cmd
if( cmd.body.docs ){
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }
]}
}, 'correct mget command');
} else {
t.deepEqual(cmd, { body: { input: 'b' }, index: 'pelias' }, 'correct suggest command');
}
});
var controller = setup( backend, mockQuery() );
var res = {
status: function( code ){
t.equal(code, 200, 'status set');
return res;
},
json: function( json ){
t.equal(typeof json, 'object', 'returns json');
t.equal(typeof json.date, 'number', 'date set');
t.equal(json.type, 'FeatureCollection', 'valid geojson');
t.true(Array.isArray(json.features), 'features is array');
t.deepEqual(json.features, expected, 'values correctly mapped');
t.end();
}
};
controller( { clean: { input: 'b' } }, res );
});
var detailed_expectation = [{
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -50.5, 100.1 ]
},
properties: {
id: 'myid1',
layer: 'mytype1',
name: 'test name1',
admin0: 'country1',
admin1: 'state1',
admin2: 'city1',
text: 'test name1, city1, state1'
}
}, {
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [ -51.5, 100.2 ]
},
properties: {
id: 'myid2',
layer: 'mytype2',
name: 'test name2',
admin0: 'country2',
admin1: 'state2',
admin2: 'city2',
text: 'test name2, city2, state2'
}
}];
test('functional success (with details)', function(t) {
var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
// the backend executes suggest (vanilla and admin-only) and mget, so we check them all based on cmd
if( cmd.body.docs ){
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }
]}
}, 'correct mget command');
} else {
t.deepEqual(cmd, { body: { input: 'b', details: true }, index: 'pelias' }, 'correct suggest command');
}
});
var controller = setup( backend, mockQuery() );
var res = {
status: function( code ){
t.equal(code, 200, 'status set');
return res;
},
json: function( json ){
t.equal(typeof json, 'object', 'returns json');
t.equal(typeof json.date, 'number', 'date set');
t.equal(json.type, 'FeatureCollection', 'valid geojson');
t.true(Array.isArray(json.features), 'features is array');
t.deepEqual(json.features, detailed_expectation, 'values with details correctly mapped');
t.end();
}
};
controller( { clean: { input: 'b', details: true } }, res );
});
};
// functionally test controller (backend failure)
module.exports.tests.functional_failure = function(test, common) {
test('functional failure', function(t) {
var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
if( cmd.body.docs ){
t.deepEqual(cmd, {
body: { docs: [
{ _id: 'mockid1', _index: 'pelias', _type: 'mocktype' },
{ _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }]
}
}, 'correct mget command');
} else if (cmd.body.layers) {
// layers are set exclusively for admin: test for admin-only layers
t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' },
'correct suggest/admin command');
} else {
t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
}
});
var controller = setup( backend, mockQuery() );
var next = function( message ){
t.equal(message,'a backend error occurred','error passed to errorHandler');
};
controller( { clean: { a: 'b' } }, undefined, next );
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('GET /suggest ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

81
test/unit/helper/queryMixer.js
Unescape View File

@ -1,81 +0,0 @@
var query_mixer = require('../../../helper/queryMixer.json');
var indeces = require('../../../query/indeces');
var alias_layers = ['poi', 'admin', 'address'];
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('interface', function(t) {
t.equal(typeof query_mixer, 'object', 'valid object');
t.equal(query_mixer.hasOwnProperty('suggest'), true, 'has suggest defined');
t.equal(query_mixer.hasOwnProperty('suggest_nearby'), true, 'has suggest_nearby defined');
t.end();
});
};
module.exports.tests.valid = function(test, common) {
var valid_keys = ['layers', 'precision', 'fuzzy'];
var valid_fuzzy_vals = ['AUTO', 0, 1, 2];
var valid_layer_vals = indeces.concat(alias_layers);
var isValidPrecision = function(t, precisionArr) {
precisionArr.forEach(function(precision) {
t.notEqual(isNaN(precision), true, precision + ' is a valid precision value');
});
};
var isValidLayer = function(t, layerArr) {
layerArr.forEach(function(this_layer) {
t.notEqual(valid_layer_vals.indexOf(this_layer), -1, 'layer value ' + this_layer + ' is valid');
});
};
var isValid = function(key, mix) {
test('valid mix (' + key + ')' , function(t) {
t.equal(keys.length > 0, true, 'valid key');
t.equal(Array.isArray( mix ), true, 'is an array');
t.equal(mix.length > 0, true, 'is not an empty array');
mix.forEach( function(this_mix) {
t.notEqual(Object.getOwnPropertyNames(this_mix).length, 0, 'object not empty');
for (var keys in this_mix) {
t.notEqual(valid_keys.indexOf(keys), -1, keys + ' is valid');
switch(keys) {
case 'fuzzy':
t.notEqual(valid_fuzzy_vals.indexOf(this_mix[keys]), -1, 'fuzzy value ' + this_mix[keys] + ' is valid');
break;
case 'layers':
t.equal(Array.isArray(this_mix[keys]), true, 'layers is an array');
t.equal(this_mix[keys].length > 0, true, 'layers is not an empty array');
isValidLayer(t, this_mix[keys]);
break;
case 'precision':
t.equal(Array.isArray( this_mix[keys] ), true, keys + ' is an array');
if (this_mix[keys].length > 0) {
isValidPrecision(t, this_mix[keys]);
}
break;
default:
break;
}
}
});
t.end();
});
};
for (var keys in query_mixer) {
isValid(keys, query_mixer[keys]);
}
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('query_mixer: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

217
test/unit/query/suggest.js
Unescape View File

@ -1,217 +0,0 @@
var generate = require('../../../query/suggest');
var queryMixer = require('../../../helper/queryMixer');
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.equal(typeof generate, 'function', 'valid function');
t.end();
});
};
module.exports.tests.query = function(test, common) {
test('valid query', function(t) {
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:1,
layers: ['test']
});
var expected = {
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
context: {
dataset: [ 'test' ],
location: {
precision: 1,
value: [ 0, 0 ]
}
},
fuzzy: { fuzziness: 0 },
}
}
};
t.deepEqual(query, expected, 'valid suggest query');
t.end();
});
test('valid query without lat/lon', function(t) {
var query = generate({
input: 'test', size: 10,
layers: ['test']
});
var expected = {
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
context: {
dataset: [ 'test' ],
location: {
precision: 1,
value: null
}
},
fuzzy: { fuzziness: 0 },
}
}
};
t.deepEqual(query, expected, 'valid suggest query');
t.end();
});
};
module.exports.tests.precision = function(test, common) {
var test_cases = [
{zoom:1, precision:1},
{zoom:2, precision:1},
{zoom:3, precision:1},
{zoom:4, precision:2},
{zoom:5, precision:2},
{zoom:6, precision:3},
{zoom:7, precision:3},
{zoom:8, precision:3},
{zoom:9, precision:3},
{zoom:10, precision:4},
{zoom:11, precision:4},
{zoom:12, precision:4},
{zoom:13, precision:4},
{zoom:14, precision:4},
{zoom:15, precision:4},
{zoom:16, precision:5},
{zoom:17, precision:5},
{zoom:18, precision:5},
{zoom:19, precision:5},
{zoom:'', precision:1},
{zoom:null, precision:1},
{zoom:undefined, precision:1}
];
test('valid precision', function(t) {
test_cases.forEach( function( test_case ){
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:test_case.zoom,
layers: ['test']
});
var expected = {
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
context: {
dataset: [ 'test' ],
location: {
precision: test_case.precision,
value: [ 0, 0 ]
}
},
fuzzy: { fuzziness: 0 },
}
}
};
t.deepEqual(query, expected, 'valid suggest query for zoom = ' + test_case.zoom);
});
t.end();
});
};
module.exports.tests.fuzziness = function(test, common) {
var test_cases = [0,1,2,'AUTO', undefined, null, ''];
test('valid fuzziness', function(t) {
test_cases.forEach( function( test_case ){
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:0,
layers: ['test']
}, undefined, test_case);
var expected = {
text: 'test',
0: {
completion: {
field: 'suggest',
size: 10,
context: {
dataset: [ 'test' ],
location: {
precision: 1,
value: [ 0, 0 ]
}
},
fuzzy: { fuzziness: test_case || 0 },
}
}
};
t.deepEqual(query, expected, 'valid suggest query for fuziness = ' + test_case);
});
t.end();
});
};
module.exports.tests.queryMixer = function(test, common) {
test('valid query mixer', function(t) {
for (var suggester in queryMixer) {
var queryMix = queryMixer[suggester];
var number_of_suggesters = queryMix.reduce(function(sum, query) {
return sum + (query.precision.length || 1);
}, 0);
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:0
}, queryMix);
// adding one to number_of_suggesters to account for the key "text" in query.
t.deepEqual(Object.keys(query).length, number_of_suggesters + 1,
suggester + ' has correct number of suggesters'
);
}
t.end();
});
};
var isValidLayer = function(t, query, layers) {
for(var qKey in query) {
var q = query[qKey];
if (q.completion) {
var query_layers = q.completion.context.dataset;
t.deepEqual(query_layers, layers, layers + ' layers set correctly');
}
}
};
module.exports.tests.layers = function(test, common) {
test('valid layers with query-mixers', function(t) {
for (var suggester in queryMixer) {
var queryMix = queryMixer[suggester];
var layers= ['geoname', 'osm'];
var query = generate({
input: 'test', size: 10,
lat: 0, lon: 0, zoom:0,
layers: layers
}, queryMix);
isValidLayer(t, query, layers);
}
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('suggest query ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

6
test/unit/run.js
Unescape View File

@ -5,24 +5,20 @@ var common = {};
var tests = [
require('./controller/index'),
require('./controller/doc'),
require('./controller/suggest'),
require('./controller/search'),
require('./service/mget'),
require('./service/search'),
require('./service/suggest'),
require('./sanitiser/suggest'),
require('./sanitiser/search'),
require('./sanitiser/reverse'),
require('./sanitiser/doc'),
require('./sanitiser/coarse'),
require('./query/indeces'),
require('./query/suggest'),
require('./query/sort'),
require('./query/search'),
require('./query/reverse'),
require('./helper/geojsonify'),
require('./helper/outputSchema'),
require('./helper/queryMixer')
require('./helper/outputSchema')
];
tests.map(function(t) {

72
test/unit/service/suggest.js
Unescape View File

@ -1,72 +0,0 @@
var setup = require('../../../service/suggest'),
mockBackend = require('../mock/backend');
var example_valid_es_query = { body: { a: 'b' }, index: 'pelias' };
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('valid interface', function(t) {
t.equal(typeof setup, 'function', 'setup is a function');
t.end();
});
};
// functionally test service
module.exports.tests.functional_success = function(test, common) {
var expected = [
[{ score: 1, text: 'mocktype:mockid1' }],
[{ score: 2, text: 'mocktype:mockid2' }]
];
test('valid ES query', function(t) {
var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
t.deepEqual(cmd, example_valid_es_query, 'no change to the command');
});
setup( backend, example_valid_es_query, function(err, data) {
t.true(Array.isArray(data), 'returns an array');
data.forEach(function(d) {
t.true(typeof d === 'object', 'valid object');
});
t.deepEqual(data, expected, 'values correctly mapped');
t.end();
});
});
};
// functionally test service
module.exports.tests.functional_failure = function(test, common) {
test('invalid ES query', function(t) {
var invalid_queries = [
{ },
{ foo: 'bar' }
];
var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
t.notDeepEqual(cmd, example_valid_es_query, 'incorrect backend command');
});
invalid_queries.forEach(function(query) {
setup( backend, [ query ], function(err, data) {
t.equal(err, 'a backend error occurred','error passed to errorHandler');
t.equal(data, undefined, 'data is undefined');
});
});
t.end();
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('SERVICE /suggest ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};
Write Preview
Loading…
Cancel
Save