mirror of https://github.com/pelias/api.git
Julian Simioni
9 years ago
38 changed files with 997 additions and 177 deletions
@ -0,0 +1,40 @@
|
||||
|
||||
var peliasQuery = require('pelias-query'), |
||||
searchDefaults = require('../search_defaults'); |
||||
|
||||
/** |
||||
This view (unfortunately) requires autocomplete to use the phrase.* index. |
||||
|
||||
ideally we wouldn't need to use this, but at time of writing we are unable |
||||
to distinguish between 'complete tokens' and 'grams' in the name.* index. |
||||
|
||||
this view was introduced in order to score exact matches higher than partial |
||||
matches, without it we find results such as "Clayton Avenue" appearing first |
||||
in the results list for the query "Clay Av". |
||||
|
||||
the view uses some of the values from the 'search_defaults.js' file to add an |
||||
additional 'SHOULD' condition which scores exact matches slighly higher |
||||
than partial matches. |
||||
**/ |
||||
|
||||
module.exports = function( vs ){ |
||||
|
||||
// make a copy of the variables so we don't interfere with the values
|
||||
// passed to other views.
|
||||
var vsCopy = new peliasQuery.Vars( vs.export() ); |
||||
|
||||
// copy phrase:* values from search defaults
|
||||
vsCopy.var('phrase:analyzer').set(searchDefaults['phrase:analyzer']); |
||||
vsCopy.var('phrase:field').set(searchDefaults['phrase:field']); |
||||
|
||||
// get a copy of the *complete* tokens produced from the input:name
|
||||
var tokens = vs.var('input:name:tokens_complete').get(); |
||||
|
||||
// no valid tokens to use, fail now, don't render this view.
|
||||
if( !tokens || tokens.length < 1 ){ return null; } |
||||
|
||||
// set 'input:name' to be only the fully completed characters
|
||||
vsCopy.var('input:name').set( tokens.join(' ') ); |
||||
|
||||
return peliasQuery.view.phrase( vsCopy ); |
||||
}; |
||||
@ -0,0 +1,17 @@
|
||||
|
||||
var peliasQuery = require('pelias-query'), |
||||
check = require('check-types'); |
||||
|
||||
/** |
||||
Population / Popularity subquery |
||||
**/ |
||||
|
||||
module.exports = function( vs ){ |
||||
|
||||
var view = peliasQuery.view.ngrams( vs ); |
||||
|
||||
view.match['name.default'].analyzer = vs.var('phrase:analyzer'); |
||||
delete view.match['name.default'].boost; |
||||
|
||||
return view; |
||||
}; |
||||
@ -0,0 +1,112 @@
|
||||
|
||||
var check = require('check-types'); |
||||
|
||||
/** |
||||
simplified version of the elaticsearch tokenizer, used in order to |
||||
be able to detect which tokens are 'complete' (user has finished typing them) |
||||
or 'incomplete' (the user has possibly only typed part of the token). |
||||
|
||||
note: we don't need to strip punctuation as that will be handled on the |
||||
elasticsearch side, so sending a token such as 'st.' is not an issue, these |
||||
tokens should *not* be modified as the anaylsis can use the punctuation to |
||||
infer meaning. |
||||
|
||||
note: this sanitizer should run *after* the '_text' sanitizer so it can |
||||
use the output of clean.parsed_text where available. |
||||
**/ |
||||
function sanitize( raw, clean ){ |
||||
|
||||
// error & warning messages
|
||||
var messages = { errors: [], warnings: [] }; |
||||
|
||||
// this is the string we will use for analysis
|
||||
var text = clean.text; |
||||
|
||||
// a boolean to track whether the input parser successfully ran; or not.
|
||||
var inputParserRanSuccessfully = false; |
||||
|
||||
// if the text parser has run then we only tokenize the 'name' section
|
||||
// of the 'parsed_text' object, ignoring the 'admin' parts.
|
||||
if( clean.hasOwnProperty('parsed_text') ) { |
||||
inputParserRanSuccessfully = true; |
||||
|
||||
// parsed_text.name is set, this is the highest priority, use this string
|
||||
if( clean.parsed_text.hasOwnProperty('name') ){ |
||||
text = clean.parsed_text.name; // use this string instead
|
||||
} |
||||
|
||||
// else handle the case where parsed_text.street was produced but
|
||||
// no parsed_text.name is produced.
|
||||
// additionally, handle the case where parsed_text.number is present
|
||||
// note: the addressit module may also produce parsed_text.unit info
|
||||
// for now, we discard that information as we don't have an appropriate
|
||||
else if( clean.parsed_text.hasOwnProperty('street') ){ |
||||
text = [ |
||||
clean.parsed_text.number, |
||||
clean.parsed_text.street |
||||
].filter(function(el){return el;}) |
||||
.join(' '); // remove empty elements
|
||||
} |
||||
} |
||||
|
||||
// always set 'clean.tokens*' arrays for consistency and to avoid upstream errors.
|
||||
clean.tokens = []; |
||||
clean.tokens_complete = []; |
||||
clean.tokens_incomplete = []; |
||||
|
||||
// sanity check that the text is valid.
|
||||
if( check.nonEmptyString( text ) ){ |
||||
|
||||
// split according to the regex used in the elasticsearch tokenizer
|
||||
// see: https://github.com/pelias/schema/blob/master/settings.js
|
||||
// see: settings.analysis.tokenizer.peliasNameTokenizer
|
||||
clean.tokens = text |
||||
.split(/[\s,\\\/]+/) // split on delimeters
|
||||
.filter(function(el){return el;}); // remove empty elements
|
||||
} |
||||
|
||||
/** |
||||
the following section splits the tokens in to two arrays called |
||||
'tokens_complete' and 'tokens_incomplete'. |
||||
|
||||
it also strips any tokens from 'tokens_incomplete' which might not |
||||
match the ngrams index (such as single grams not stored in the index). |
||||
**/ |
||||
|
||||
// split the tokens in to 'complete' and 'incomplete'.
|
||||
if( clean.tokens.length ){ |
||||
|
||||
// if all the tokens are complete, simply copy them from clean.tokens
|
||||
if( inputParserRanSuccessfully ){ |
||||
|
||||
// all these tokens are complete!
|
||||
clean.tokens_complete = clean.tokens.slice(); |
||||
|
||||
// user hasn't finished typing yet
|
||||
} else { |
||||
|
||||
// make a copy of the tokens and remove the last element
|
||||
var tokensCopy = clean.tokens.slice(), |
||||
lastToken = tokensCopy.pop(); |
||||
|
||||
// set all but the last token as 'complete'
|
||||
clean.tokens_complete = tokensCopy; |
||||
|
||||
/** |
||||
if the last token is a single non-numeric character then we must discard it. |
||||
|
||||
at time of writing, single non-numeric ngrams are not stored in the index, |
||||
sending them as part of the query would result in 0 documents being returned. |
||||
**/ |
||||
if( lastToken && ( lastToken.length > 1 || lastToken.match(/[0-9]/) ) ){ |
||||
clean.tokens_incomplete = [ lastToken ]; |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
return messages; |
||||
} |
||||
|
||||
// export function
|
||||
module.exports = sanitize; |
||||
@ -0,0 +1,155 @@
|
||||
|
||||
module.exports = { |
||||
'query': { |
||||
'filtered': { |
||||
'query': { |
||||
'bool': { |
||||
'must': [{ |
||||
'match': { |
||||
'name.default': { |
||||
'analyzer': 'peliasQueryFullToken', |
||||
'type': 'phrase', |
||||
'boost': 1, |
||||
'slop': 3, |
||||
'query': 'k road' |
||||
} |
||||
} |
||||
}], |
||||
'should':[ |
||||
{ |
||||
'match': { |
||||
'address_parts.street': { |
||||
'query': 'k road', |
||||
'boost': 5, |
||||
'analyzer': 'peliasStreet' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.country': { |
||||
'query': 'laird', |
||||
'boost': 800, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.region': { |
||||
'query': 'laird', |
||||
'boost': 600, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.region_a': { |
||||
'query': 'laird', |
||||
'boost': 600, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.county': { |
||||
'query': 'laird', |
||||
'boost': 400, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.borough': { |
||||
'analyzer': 'peliasAdmin', |
||||
'boost': 600, |
||||
'query': 'laird' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.localadmin': { |
||||
'query': 'laird', |
||||
'boost': 200, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.locality': { |
||||
'query': 'laird', |
||||
'boost': 200, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, { |
||||
'match': { |
||||
'parent.neighbourhood': { |
||||
'query': 'laird', |
||||
'boost': 200, |
||||
'analyzer': 'peliasAdmin' |
||||
} |
||||
} |
||||
}, |
||||
{ |
||||
'match': { |
||||
'phrase.default': { |
||||
'analyzer' : 'peliasPhrase', |
||||
'type' : 'phrase', |
||||
'boost' : 1, |
||||
'slop' : 3, |
||||
'query' : 'k road' |
||||
} |
||||
} |
||||
}, |
||||
{ |
||||
'function_score': { |
||||
'query': { |
||||
'match': { |
||||
'name.default': { |
||||
'analyzer': 'peliasQueryFullToken', |
||||
'query': 'k road', |
||||
} |
||||
} |
||||
}, |
||||
'max_boost': 20, |
||||
'score_mode': 'first', |
||||
'boost_mode': 'replace', |
||||
'functions': [{ |
||||
'field_value_factor': { |
||||
'modifier': 'log1p', |
||||
'field': 'popularity', |
||||
'missing': 1 |
||||
}, |
||||
'weight': 1 |
||||
}] |
||||
} |
||||
},{ |
||||
'function_score': { |
||||
'query': { |
||||
'match': { |
||||
'name.default': { |
||||
'analyzer': 'peliasQueryFullToken', |
||||
'query': 'k road', |
||||
} |
||||
} |
||||
}, |
||||
'max_boost': 20, |
||||
'score_mode': 'first', |
||||
'boost_mode': 'replace', |
||||
'functions': [{ |
||||
'field_value_factor': { |
||||
'modifier': 'log1p', |
||||
'field': 'population', |
||||
'missing': 1 |
||||
}, |
||||
'weight': 3 |
||||
}] |
||||
} |
||||
}] |
||||
} |
||||
} |
||||
} |
||||
}, |
||||
'sort': [ '_score' ], |
||||
'size': 20, |
||||
'track_scores': true |
||||
}; |
||||
@ -0,0 +1,457 @@
|
||||
var sanitiser = require('../../../sanitiser/_tokenizer'); |
||||
|
||||
module.exports.tests = {}; |
||||
|
||||
module.exports.tests.sanity_checks = function(test, common) { |
||||
test('clean.text not set', function(t) { |
||||
|
||||
var clean = {}; // clean.text not set
|
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// no tokens produced
|
||||
t.deepEquals(clean.tokens, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_complete, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('clean.text not a string', function(t) { |
||||
|
||||
var clean = { text: {} }; // clean.text not a string
|
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// no tokens produced
|
||||
t.deepEquals(clean.tokens, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_complete, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('empty string', function(t) { |
||||
|
||||
var clean = { text: '' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// no tokens produced
|
||||
t.deepEquals(clean.tokens, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_complete, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('clean.parsed_text set but clean.parsed_text.name invalid', function(t) { |
||||
|
||||
var clean = { parsed_text: { text: {} } }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// no tokens produced
|
||||
t.deepEquals(clean.tokens, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_complete, [], 'no tokens'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('favor clean.parsed_text.name over clean.text', function(t) { |
||||
|
||||
var clean = { parsed_text: { name: 'foo' }, text: 'bar' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// favor clean.parsed_text.name over clean.text
|
||||
t.deepEquals(clean.tokens, [ 'foo' ], 'use clean.parsed_text.name'); |
||||
t.deepEquals(clean.tokens_complete, [ 'foo' ], 'use clean.parsed_text.name'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('favor clean.parsed_text street data over clean.text', function(t) { |
||||
|
||||
var clean = { parsed_text: { number: '190', street: 'foo st' }, text: 'bar' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// favor clean.parsed_text.name over clean.text
|
||||
t.deepEquals(clean.tokens, [ '190', 'foo', 'st' ], 'use street name + number'); |
||||
t.deepEquals(clean.tokens_complete, [ '190', 'foo', 'st' ], 'use street name + number'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('favor clean.parsed_text.name over clean.parsed_text street data', function(t) { |
||||
|
||||
var clean = { parsed_text: { number: '190', street: 'foo st', name: 'foo' }, text: 'bar' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// favor clean.parsed_text.name over all other variables
|
||||
t.deepEquals(clean.tokens, [ 'foo' ], 'use clean.parsed_text.name'); |
||||
t.deepEquals(clean.tokens_complete, [ 'foo' ], 'use clean.parsed_text.name'); |
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.space_delimiter = function(test, common) { |
||||
test('space delimiter - simple', function(t) { |
||||
|
||||
var clean = { text: '30 west 26th street new york' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new', |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('space delimiter - multiple spaces / other whitespace', function(t) { |
||||
|
||||
var clean = { text: ' 30 west \t26th \nstreet new york ' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new', |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.comma_delimiter = function(test, common) { |
||||
test('comma delimiter - simple', function(t) { |
||||
|
||||
var clean = { text: '30 west 26th street, new york' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new', |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('comma delimiter - multiple commas', function(t) { |
||||
|
||||
var clean = { text: ',30 west 26th street,,, new york,' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new', |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'30', |
||||
'west', |
||||
'26th', |
||||
'street', |
||||
'new' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'york' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.forward_slash_delimiter = function(test, common) { |
||||
test('forward slash delimiter - simple', function(t) { |
||||
|
||||
var clean = { text: 'Bedell Street/133rd Avenue' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd', |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('forward slash - multiple slashes', function(t) { |
||||
|
||||
var clean = { text: '/Bedell Street//133rd Avenue/' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd', |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd' |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.final_token_single_gram = function(test, common) { |
||||
test('final token single gram - numeric', function(t) { |
||||
|
||||
var clean = { text: 'grolmanstrasse 1' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'grolmanstrasse', |
||||
'1' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'grolmanstrasse', |
||||
], 'tokens produced'); |
||||
|
||||
// last token marked as 'incomplete'
|
||||
t.deepEquals(clean.tokens_incomplete, [ |
||||
'1' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('final token single gram - non-numeric', function(t) { |
||||
|
||||
var clean = { text: 'grolmanstrasse a' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'grolmanstrasse', |
||||
'a' |
||||
], 'tokens produced'); |
||||
|
||||
// all but last token marked as 'complete'
|
||||
t.deepEquals(clean.tokens_complete, [ |
||||
'grolmanstrasse', |
||||
], 'tokens produced'); |
||||
|
||||
// last token removed!
|
||||
t.deepEquals(clean.tokens_incomplete, [], 'no tokens'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.back_slash_delimiter = function(test, common) { |
||||
test('back slash delimiter - simple', function(t) { |
||||
|
||||
var clean = { text: 'Bedell Street\\133rd Avenue' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd', |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
test('back slash - multiple slashes', function(t) { |
||||
|
||||
var clean = { text: '\\Bedell Street\\\\133rd Avenue\\' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd', |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.tests.mixed_delimiter = function(test, common) { |
||||
test('mixed delimiters', function(t) { |
||||
|
||||
var clean = { text: ',/Bedell Street\\, \n\t ,\\//133rd Avenue, /\n/' }; |
||||
var messages = sanitiser({}, clean); |
||||
|
||||
// tokens produced
|
||||
t.deepEquals(clean.tokens, [ |
||||
'Bedell', |
||||
'Street', |
||||
'133rd', |
||||
'Avenue' |
||||
], 'tokens produced'); |
||||
|
||||
// no errors/warnings produced
|
||||
t.deepEquals(messages.errors, [], 'no errors'); |
||||
t.deepEquals(messages.warnings, [], 'no warnings'); |
||||
|
||||
t.end(); |
||||
}); |
||||
}; |
||||
|
||||
module.exports.all = function (tape, common) { |
||||
function test(name, testFunction) { |
||||
return tape('SANITISER _tokenizer: ' + name, testFunction); |
||||
} |
||||
|
||||
for( var testCase in module.exports.tests ){ |
||||
module.exports.tests[testCase](test, common); |
||||
} |
||||
}; |
||||
Loading…
Reference in new issue