mirror of https://github.com/pelias/api.git
Stephen K Hess
8 years ago
committed by
GitHub
21 changed files with 620 additions and 64 deletions
@ -0,0 +1,118 @@
|
||||
/** |
||||
* |
||||
* Basic confidence score should be computed and returned for each item in the results. |
||||
* The score should range between 0-1, and take into consideration as many factors as possible. |
||||
* |
||||
* Some factors to consider: |
||||
* |
||||
* - number of results from ES |
||||
* - fallback status (aka layer match between expected and actual) |
||||
*/ |
||||
|
||||
var check = require('check-types'); |
||||
var logger = require('pelias-logger').get('api-confidence'); |
||||
|
||||
function setup() { |
||||
return computeScores; |
||||
} |
||||
|
||||
function computeScores(req, res, next) { |
||||
// do nothing if no result data set or if the query is not of the fallback variety
|
||||
// later add disambiguation to this list
|
||||
if (check.undefined(req.clean) || check.undefined(res) || |
||||
check.undefined(res.data) || check.undefined(res.meta) || |
||||
res.meta.query_type !== 'fallback') { |
||||
return next(); |
||||
} |
||||
|
||||
// loop through data items and determine confidence scores
|
||||
res.data = res.data.map(computeConfidenceScore.bind(null, req)); |
||||
|
||||
next(); |
||||
} |
||||
|
||||
/** |
||||
* Check all types of things to determine how confident we are that this result |
||||
* is correct. |
||||
* |
||||
* @param {object} req |
||||
* @param {object} hit |
||||
* @returns {object} |
||||
*/ |
||||
function computeConfidenceScore(req, hit) { |
||||
|
||||
// if parsed text doesn't exist, which it never should, just assign a low confidence and move on
|
||||
if (!req.clean.hasOwnProperty('parsed_text')) { |
||||
hit.confidence = 0.1; |
||||
hit.match_type = 'unknown'; |
||||
return hit; |
||||
} |
||||
|
||||
// start with a confidence level of 1 because we trust ES queries to be accurate
|
||||
hit.confidence = 1.0; |
||||
|
||||
// in the case of fallback there might be deductions
|
||||
hit.confidence *= checkFallbackLevel(req, hit); |
||||
|
||||
// truncate the precision
|
||||
hit.confidence = Number((hit.confidence).toFixed(3)); |
||||
|
||||
return hit; |
||||
} |
||||
|
||||
function checkFallbackLevel(req, hit) { |
||||
if (checkFallbackOccurred(req, hit)) { |
||||
hit.match_type = 'fallback'; |
||||
|
||||
// if we know a fallback occurred, deduct points based on layer granularity
|
||||
switch (hit.layer) { |
||||
case 'venue': |
||||
case 'address': |
||||
logger.warn('Fallback scenarios should not result in address or venue records!', req.clean.parsed_text); |
||||
return 0.8; |
||||
case 'street': |
||||
return 0.8; |
||||
case 'locality': |
||||
case 'borough': |
||||
case 'neighbourhood': |
||||
return 0.6; |
||||
case 'macrocounty': |
||||
case 'county': |
||||
case 'localadmin': |
||||
return 0.4; |
||||
case 'region': |
||||
return 0.3; |
||||
case 'country': |
||||
case 'dependency': |
||||
case 'macroregion': |
||||
return 0.1; |
||||
default: |
||||
return 0.1; |
||||
} |
||||
} |
||||
|
||||
hit.match_type = 'exact'; |
||||
return 1.0; |
||||
} |
||||
|
||||
function checkFallbackOccurred(req, hit) { |
||||
// at this time we only do this for address queries, so keep this simple
|
||||
// TODO: add other layer checks once we start handling disambiguation
|
||||
|
||||
return (requestedAddress(req) && hit.layer !== 'address') || |
||||
(requestedStreet(req) && hit.layer !== 'street'); |
||||
} |
||||
|
||||
function requestedAddress(req) { |
||||
// house number and street name were specified
|
||||
return req.clean.parsed_text.hasOwnProperty('number') && |
||||
req.clean.parsed_text.hasOwnProperty('street'); |
||||
} |
||||
|
||||
function requestedStreet(req) { |
||||
// only street name was specified
|
||||
return !req.clean.parsed_text.hasOwnProperty('number') && |
||||
req.clean.parsed_text.hasOwnProperty('street'); |
||||
} |
||||
|
||||
module.exports = setup; |
||||
@ -0,0 +1,250 @@
|
||||
var confidenceScore = require('../../../middleware/confidenceScoreFallback')(); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.confidenceScore = function(test, common) { |
||||
|
||||
test('empty res and req should not throw exception', function(t) { |
||||
function testIt() { |
||||
confidenceScore({}, {}, function() {}); |
||||
} |
||||
|
||||
t.doesNotThrow(testIt, 'an exception should not have been thrown'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('res.results without parsed_text should not throw exception', function(t) { |
||||
var req = {}; |
||||
var res = { |
||||
data: [{ |
||||
name: 'foo' |
||||
}], |
||||
meta: [10] |
||||
}; |
||||
|
||||
function testIt() { |
||||
confidenceScore(req, res, function() {}); |
||||
} |
||||
|
||||
t.doesNotThrow(testIt, 'an exception should not have been thrown'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('hit without address should not error', function(t) { |
||||
var req = { |
||||
clean: { |
||||
text: 'test name3', |
||||
parsed_text: { |
||||
postalcode: 12345 |
||||
} |
||||
} |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
name: { |
||||
default: 'foo' |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'original' |
||||
} |
||||
}; |
||||
|
||||
function testIt() { |
||||
confidenceScore(req, res, function() {}); |
||||
} |
||||
|
||||
t.doesNotThrow(testIt, 'an exception should not have been thrown with no address'); |
||||
t.end(); |
||||
}); |
||||
|
||||
|
||||
test('res.results without parsed_text should not throw exception', function(t) { |
||||
var req = { |
||||
clean: { text: 'test name1' } |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
_score: 10, |
||||
found: true, |
||||
value: 1, |
||||
center_point: { lat: 100.1, lon: -50.5 }, |
||||
name: { default: 'test name1' }, |
||||
parent: { |
||||
country: ['country1'], |
||||
region: ['state1'], |
||||
county: ['city1'] |
||||
} |
||||
}, { |
||||
_score: 20, |
||||
value: 2, |
||||
center_point: { lat: 100.2, lon: -51.5 }, |
||||
name: { default: 'test name2' }, |
||||
parent: { |
||||
country: ['country2'], |
||||
region: ['state2'], |
||||
county: ['city2'] |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'fallback' |
||||
} |
||||
}; |
||||
|
||||
confidenceScore(req, res, function() {}); |
||||
t.equal(res.data[0].confidence, 0.1, 'score was set'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('no fallback addresses should have max score', function(t) { |
||||
var req = { |
||||
clean: { |
||||
text: '123 Main St, City, NM', |
||||
parsed_text: { |
||||
number: 123, |
||||
street: 'Main St', |
||||
state: 'NM' |
||||
} |
||||
} |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
_score: 10, |
||||
found: true, |
||||
value: 1, |
||||
layer: 'address', |
||||
center_point: { lat: 100.1, lon: -50.5 }, |
||||
name: { default: 'test name1' }, |
||||
parent: { |
||||
country: ['country1'], |
||||
region: ['region1'], |
||||
county: ['city1'] |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'fallback' |
||||
} |
||||
}; |
||||
|
||||
confidenceScore(req, res, function() {}); |
||||
t.equal(res.data[0].confidence, 1.0, 'max score was set'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('no fallback street query should have max score', function(t) { |
||||
var req = { |
||||
clean: { |
||||
text: 'Main St, City, NM', |
||||
parsed_text: { |
||||
street: 'Main St', |
||||
state: 'NM' |
||||
} |
||||
} |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
_score: 10, |
||||
found: true, |
||||
value: 1, |
||||
layer: 'street', |
||||
center_point: { lat: 100.1, lon: -50.5 }, |
||||
name: { default: 'test name1' }, |
||||
parent: { |
||||
country: ['country1'], |
||||
region: ['region1'], |
||||
county: ['city1'] |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'fallback' |
||||
} |
||||
}; |
||||
|
||||
confidenceScore(req, res, function() {}); |
||||
t.equal(res.data[0].confidence, 1.0, 'max score was set'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('fallback to locality should have score deduction', function(t) { |
||||
var req = { |
||||
clean: { |
||||
text: '123 Main St, City, NM', |
||||
parsed_text: { |
||||
number: 123, |
||||
street: 'Main St', |
||||
state: 'NM' |
||||
} |
||||
} |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
_score: 10, |
||||
found: true, |
||||
value: 1, |
||||
layer: 'locality', |
||||
center_point: { lat: 100.1, lon: -50.5 }, |
||||
name: { default: 'test name1' }, |
||||
parent: { |
||||
country: ['country1'] |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'fallback' |
||||
} |
||||
}; |
||||
|
||||
confidenceScore(req, res, function() {}); |
||||
t.equal(res.data[0].confidence, 0.6, 'score was set'); |
||||
t.end(); |
||||
}); |
||||
|
||||
test('fallback to country should have score deduction', function(t) { |
||||
var req = { |
||||
clean: { |
||||
text: '123 Main St, City, NM, USA', |
||||
parsed_text: { |
||||
number: 123, |
||||
street: 'Main St', |
||||
state: 'NM', |
||||
country: 'USA' |
||||
} |
||||
} |
||||
}; |
||||
var res = { |
||||
data: [{ |
||||
_score: 10, |
||||
found: true, |
||||
value: 1, |
||||
layer: 'country', |
||||
center_point: { lat: 100.1, lon: -50.5 }, |
||||
name: { default: 'test name1' }, |
||||
parent: { |
||||
country: ['country1'] |
||||
} |
||||
}], |
||||
meta: { |
||||
scores: [10], |
||||
query_type: 'fallback' |
||||
} |
||||
}; |
||||
|
||||
confidenceScore(req, res, function() {}); |
||||
t.equal(res.data[0].confidence, 0.1, 'score was set'); |
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('[middleware] confidenceScore: ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue