mirror of https://github.com/pelias/api.git
Julian Simioni
6 years ago
committed by
GitHub
16 changed files with 655 additions and 13 deletions
@ -0,0 +1,121 @@
|
||||
/** |
||||
This view allows users to specify a custom boost for sources and layers. |
||||
|
||||
The view is implemented using a 'function_score' query, which enumerates multiple 'functions', each |
||||
function will assign a 'score' to each document when matched. |
||||
|
||||
A document can match more than one function, in this case the 'score_mode' is used to decide how these |
||||
scores are combined, the default is 'sum'. |
||||
|
||||
Likewise, a document can also match zero functions, in this case it is assigned a score of 'min_score'. |
||||
|
||||
The computed score is then multiplied by the 'boost' value in order to come up with the final boost value |
||||
which will be assigned to that document. The 'boost' value is essentially a hard-coded multiplier for the score. |
||||
|
||||
The 'max_boost' property is simply a ceiling for this computed boost, if the computed boosted is higher than |
||||
max_boost it will be assigned the value of max_boost instead. |
||||
|
||||
Note: This is a simple use of the 'function_score' query, as such we don't use the 'boost_mode' property |
||||
(because there is no query section) and the 'weight' values we assign are simply returned verbatim |
||||
(because we use filter queries for the function scoring). |
||||
|
||||
ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html
|
||||
|
||||
example config section: |
||||
{ |
||||
"source": { |
||||
"openstreetmap": 5 |
||||
}, |
||||
"layer": { |
||||
"street": 3, |
||||
"country": 5 |
||||
} |
||||
} |
||||
|
||||
example query: |
||||
{ |
||||
"function_score": { |
||||
"query": { |
||||
"match_all": {} |
||||
}, |
||||
"functions": [{ |
||||
"filter": { |
||||
"match": { |
||||
"layer": "intersections" |
||||
} |
||||
}, |
||||
"weight": 1.6 |
||||
},{ |
||||
"filter": { |
||||
"match": { |
||||
"layer": "stops" |
||||
} |
||||
}, |
||||
"weight": 2.4 |
||||
}], |
||||
"boost": 5, |
||||
"max_boost": 40, |
||||
"score_mode": "sum", |
||||
"boost_mode": "multiply", |
||||
"min_score": 1 |
||||
} |
||||
} |
||||
**/ |
||||
|
||||
// supported top-level config items
|
||||
const TARGETS = ['source', 'layer']; |
||||
|
||||
module.exports = function( config ) { |
||||
|
||||
// no valid config to use, fail now, don't render this view.
|
||||
if( !config ) { return function(){ return null; }; } |
||||
|
||||
return function( vs ) { |
||||
|
||||
// validate required params
|
||||
if( !vs.isset('custom:boosting:min_score') || |
||||
!vs.isset('custom:boosting:boost') || |
||||
!vs.isset('custom:boosting:max_boost') || |
||||
!vs.isset('custom:boosting:score_mode') || |
||||
!vs.isset('custom:boosting:boost_mode') ){ |
||||
return null; |
||||
} |
||||
|
||||
// base 'function_score' view
|
||||
var view = { |
||||
'function_score': { |
||||
'query': { 'match_all': {} }, // apply to all documents
|
||||
'functions': [], // a list of functions which contribute to a 'score' for each document
|
||||
'min_score': vs.var('custom:boosting:min_score'), |
||||
'boost': vs.var('custom:boosting:boost'), |
||||
'max_boost': vs.var('custom:boosting:max_boost'), |
||||
'score_mode': vs.var('custom:boosting:score_mode'), |
||||
'boost_mode': vs.var('custom:boosting:boost_mode') |
||||
}, |
||||
}; |
||||
|
||||
// iterate over supported targets and their values
|
||||
TARGETS.forEach( function( target ) { |
||||
if( 'object' === typeof config[target] ) { |
||||
Object.keys(config[target]).forEach(function(value) { |
||||
|
||||
// add a scoring function for this target, assigning a weight
|
||||
let weight = config[target][value]; |
||||
view.function_score.functions.push({ |
||||
'weight': isNaN(weight) ? 1 : weight, |
||||
'filter': { |
||||
'match': { |
||||
[target]: value |
||||
} |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
}); |
||||
|
||||
// no functions were generated, fail now, don't render this view.
|
||||
if( view.function_score.functions.length === 0 ) { return null; } |
||||
|
||||
return view; |
||||
}; |
||||
}; |
||||
@ -0,0 +1,106 @@
|
||||
{ |
||||
"type": "autocomplete", |
||||
"body": { |
||||
"query": { |
||||
"bool": { |
||||
"must": [ |
||||
{ |
||||
"match": { |
||||
"name.default": { |
||||
"analyzer": "peliasQueryFullToken", |
||||
"type": "phrase", |
||||
"boost": 1, |
||||
"slop": 3, |
||||
"query": "foo" |
||||
} |
||||
} |
||||
} |
||||
], |
||||
"should": [ |
||||
{ |
||||
"match": { |
||||
"phrase.default": { |
||||
"analyzer": "peliasPhrase", |
||||
"type": "phrase", |
||||
"boost": 1, |
||||
"slop": 3, |
||||
"query": "foo" |
||||
} |
||||
} |
||||
}, |
||||
{ |
||||
"function_score": { |
||||
"query": { |
||||
"match_all": {} |
||||
}, |
||||
"max_boost": 20, |
||||
"functions": [ |
||||
{ |
||||
"field_value_factor": { |
||||
"modifier": "log1p", |
||||
"field": "popularity", |
||||
"missing": 1 |
||||
}, |
||||
"weight": 1 |
||||
} |
||||
], |
||||
"score_mode": "first", |
||||
"boost_mode": "replace" |
||||
} |
||||
}, |
||||
{ |
||||
"function_score": { |
||||
"query": { |
||||
"match_all": {} |
||||
}, |
||||
"max_boost": 20, |
||||
"functions": [ |
||||
{ |
||||
"field_value_factor": { |
||||
"modifier": "log1p", |
||||
"field": "population", |
||||
"missing": 1 |
||||
}, |
||||
"weight": 3 |
||||
} |
||||
], |
||||
"score_mode": "first", |
||||
"boost_mode": "replace" |
||||
} |
||||
},{ |
||||
"function_score": { |
||||
"query": { |
||||
"match_all": {} |
||||
}, |
||||
"min_score": 1, |
||||
"boost": 5, |
||||
"max_boost": 50, |
||||
"score_mode": "sum", |
||||
"boost_mode": "multiply", |
||||
"functions": [{ |
||||
"filter": { |
||||
"match": { |
||||
"source": "openstreetmap" |
||||
} |
||||
}, |
||||
"weight": 5 |
||||
},{ |
||||
"filter": { |
||||
"match": { |
||||
"layer": "transit" |
||||
} |
||||
}, |
||||
"weight": 3 |
||||
}] |
||||
} |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"size": 20, |
||||
"track_scores": true, |
||||
"sort": [ |
||||
"_score" |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,108 @@
|
||||
{ |
||||
"type": "search_original", |
||||
"body": { |
||||
"query": { |
||||
"bool": { |
||||
"must": [{ |
||||
"match": { |
||||
"name.default": { |
||||
"query": "test", |
||||
"boost": 1, |
||||
"analyzer": "peliasQueryFullToken" |
||||
} |
||||
} |
||||
}], |
||||
"should": [{ |
||||
"match": { |
||||
"phrase.default": { |
||||
"query": "test", |
||||
"analyzer": "peliasPhrase", |
||||
"type": "phrase", |
||||
"boost": 1, |
||||
"slop": 2 |
||||
} |
||||
} |
||||
},{ |
||||
"function_score": { |
||||
"query": { |
||||
"match": { |
||||
"phrase.default": { |
||||
"query": "test", |
||||
"analyzer": "peliasPhrase", |
||||
"type": "phrase", |
||||
"slop": 2, |
||||
"boost": 1 |
||||
} |
||||
} |
||||
}, |
||||
"max_boost": 20, |
||||
"score_mode": "first", |
||||
"boost_mode": "replace", |
||||
"functions": [{ |
||||
"field_value_factor": { |
||||
"modifier": "log1p", |
||||
"field": "popularity", |
||||
"missing": 1 |
||||
}, |
||||
"weight": 1 |
||||
}] |
||||
} |
||||
},{ |
||||
"function_score": { |
||||
"query": { |
||||
"match": { |
||||
"phrase.default": { |
||||
"query": "test", |
||||
"analyzer": "peliasPhrase", |
||||
"type": "phrase", |
||||
"slop": 2, |
||||
"boost": 1 |
||||
} |
||||
} |
||||
}, |
||||
"max_boost": 20, |
||||
"score_mode": "first", |
||||
"boost_mode": "replace", |
||||
"functions": [{ |
||||
"field_value_factor": { |
||||
"modifier": "log1p", |
||||
"field": "population", |
||||
"missing": 1 |
||||
}, |
||||
"weight": 2 |
||||
}] |
||||
} |
||||
},{ |
||||
"function_score": { |
||||
"query": { |
||||
"match_all": {} |
||||
}, |
||||
"min_score": 1, |
||||
"boost": 5, |
||||
"max_boost": 50, |
||||
"score_mode": "sum", |
||||
"boost_mode": "multiply", |
||||
"functions": [{ |
||||
"filter": { |
||||
"match": { |
||||
"source": "openstreetmap" |
||||
} |
||||
}, |
||||
"weight": 5 |
||||
},{ |
||||
"filter": { |
||||
"match": { |
||||
"layer": "transit" |
||||
} |
||||
}, |
||||
"weight": 3 |
||||
}] |
||||
} |
||||
}] |
||||
} |
||||
}, |
||||
"sort": [ "_score" ], |
||||
"size": 10, |
||||
"track_scores": true |
||||
} |
||||
} |
||||
@ -0,0 +1,54 @@
|
||||
const proxyquire = require('proxyquire'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.query = function(test, common) { |
||||
test('valid autocomplete with custom boosts', function(t) { |
||||
const clean = { |
||||
tokens: ['foo'], |
||||
tokens_complete: ['foo'], |
||||
tokens_incomplete: [], |
||||
text: 'test', |
||||
querySize: 10 |
||||
}; |
||||
|
||||
const config_with_boosts = { |
||||
generate: function() { |
||||
return { |
||||
api: { |
||||
customBoosts: { |
||||
source: { |
||||
openstreetmap: 5 |
||||
}, |
||||
layer: { |
||||
transit: 3 |
||||
} |
||||
} |
||||
} |
||||
}; |
||||
} |
||||
}; |
||||
|
||||
var expected_query = require('../fixture/autocomplete_custom_boosts.json'); |
||||
|
||||
const autocomplete_query_module = proxyquire('../../../query/autocomplete', { |
||||
'pelias-config': config_with_boosts |
||||
}); |
||||
|
||||
const actual_query = JSON.parse( JSON.stringify( autocomplete_query_module(clean) ) ); |
||||
|
||||
t.deepEqual(actual_query, expected_query, 'query as expected'); |
||||
t.pass(); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('autocomplete with custom boosts query ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
@ -0,0 +1,53 @@
|
||||
const proxyquire = require('proxyquire'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.query = function(test, common) { |
||||
test('valid search with custom boosts', function(t) { |
||||
const clean = { |
||||
tokens: ['foo'], |
||||
tokens_complete: ['foo'], |
||||
tokens_incomplete: [], |
||||
text: 'test', |
||||
querySize: 10 |
||||
}; |
||||
|
||||
const config_with_boosts = { |
||||
generate: function() { |
||||
return { |
||||
api: { |
||||
customBoosts: { |
||||
source: { |
||||
openstreetmap: 5 |
||||
}, |
||||
layer: { |
||||
transit: 3 |
||||
} |
||||
} |
||||
} |
||||
}; |
||||
} |
||||
}; |
||||
|
||||
var expected_query = require('../fixture/search_with_custom_boosts.json'); |
||||
|
||||
const search_query_module = proxyquire('../../../query/search_original', { |
||||
'pelias-config': config_with_boosts |
||||
}); |
||||
|
||||
const actual_query = JSON.parse( JSON.stringify( search_query_module(clean) ) ); |
||||
t.deepEqual(actual_query, expected_query, 'query as expected'); |
||||
t.pass(); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('search with custom boosts query ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
@ -0,0 +1,119 @@
|
||||
const query = require('pelias-query'); |
||||
const vs = new query.Vars(require('../../../../query/search_defaults')); |
||||
const boost_sources_and_layers = require('../../../../query/view/boost_sources_and_layers'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.empty_config = function(test, common) { |
||||
test('empty configuration returns empty query', function(t) { |
||||
const view = boost_sources_and_layers({}); |
||||
const rendered = view(vs); |
||||
t.equal(rendered, null, 'query is empty'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('undefined configuration returns empty query', function(t) { |
||||
const view = boost_sources_and_layers(undefined); |
||||
const rendered = view(vs); |
||||
t.equal(rendered, null, 'query is empty'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.single_item_config = function(test, common) { |
||||
test('config with single layer entry produces a single scoring function with weight', function(t) { |
||||
const config = { |
||||
layer: { |
||||
locality: 5 |
||||
} |
||||
}; |
||||
const expected_query = { |
||||
'function_score': { |
||||
'query': { |
||||
'match_all': {} |
||||
}, |
||||
'functions': [{ |
||||
'filter': { |
||||
'match': { |
||||
'layer': 'locality' |
||||
} |
||||
}, |
||||
'weight': 5 |
||||
}], |
||||
'boost': vs.var('custom:boosting:boost'), |
||||
'max_boost': vs.var('custom:boosting:max_boost'), |
||||
'score_mode': vs.var('custom:boosting:score_mode'), |
||||
'boost_mode': vs.var('custom:boosting:boost_mode'), |
||||
'min_score': vs.var('custom:boosting:min_score') |
||||
} |
||||
}; |
||||
|
||||
const view = boost_sources_and_layers(config); |
||||
|
||||
t.deepEquals(view(vs), expected_query, 'query contains a single scoring function'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.mulitple_item_config = function(test, common) { |
||||
test('config with multiple items produces multiple scoring functions', function(t) { |
||||
const config = { |
||||
source: { |
||||
whosonfirst: 6 |
||||
}, |
||||
layer: { |
||||
country: 2, |
||||
borough: 0.5 |
||||
}, |
||||
}; |
||||
const expected_query = { |
||||
'function_score': { |
||||
'query': { |
||||
'match_all': {} |
||||
}, |
||||
'functions': [{ |
||||
'filter': { |
||||
'match': { |
||||
'source': 'whosonfirst' |
||||
} |
||||
}, |
||||
'weight': 6 |
||||
},{ |
||||
'filter': { |
||||
'match': { |
||||
'layer': 'country' |
||||
} |
||||
}, |
||||
'weight': 2 |
||||
},{ |
||||
'filter': { |
||||
'match': { |
||||
'layer': 'borough' |
||||
} |
||||
}, |
||||
'weight': 0.5 |
||||
}], |
||||
'boost': vs.var('custom:boosting:boost'), |
||||
'max_boost': vs.var('custom:boosting:max_boost'), |
||||
'score_mode': vs.var('custom:boosting:score_mode'), |
||||
'boost_mode': vs.var('custom:boosting:boost_mode'), |
||||
'min_score': vs.var('custom:boosting:min_score') |
||||
} |
||||
}; |
||||
const view = boost_sources_and_layers(config); |
||||
|
||||
t.deepEquals(view(vs), expected_query, 'query contains multiple scoring functions'); |
||||
t.end(); |
||||
|
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('boost sources and layers ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue