allow single non-numeric chars, change boost_mode to 'replace' for function_score queries

query/autocomplete_defaults.js

@@ -34,7 +34,7 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'focus:weight': 40,
 
   'function_score:score_mode': 'avg',
-  'function_score:boost_mode': 'multiply',
+  'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
   'address:housenumber:field': 'address_parts.number',

query/view/ngrams_strict.js

@@ -10,10 +10,16 @@ var peliasQuery = require('pelias-query');
 
 module.exports = function( vs ){
 
+  // validate required params
+  if( !vs.isset('phrase:slop') ){
+    return null;
+  }
+
   var view = peliasQuery.view.ngrams( vs );
 
   view.match['name.default'].type = 'phrase';
   view.match['name.default'].operator = 'and';
+  view.match['name.default'].slop = vs.var('phrase:slop');
 
   return view;
 };

query/view/pop_subquery.js

@@ -4,14 +4,13 @@ var peliasQuery = require('pelias-query'),
 
 /**
   Population / Popularity subquery
-**/
-
-module.exports = function( vs ){
 
-  var view = peliasQuery.view.ngrams( vs );
+  In prior versions we have had restricted the population/popularity boost
+  to only a section of the query results.
 
-  view.match['name.default'].analyzer = vs.var('phrase:analyzer');
+  Currently it is configured to `match_all`, ie. targets all records.
-  delete view.match['name.default'].boost;
+**/
 
-  return view;
+module.exports = function( vs ){
+  return { 'match_all': {} };
 };

sanitiser/_tokenizer.js

@@ -92,13 +92,7 @@ function sanitize( raw, clean ){
       // set all but the last token as 'complete'
       clean.tokens_complete = tokensCopy;
 
-      /**
+      if( lastToken ){
-        if the last token is a single non-numeric character then we must discard it.
-
-        at time of writing, single non-numeric ngrams are not stored in the index,
-        sending them as part of the query would result in 0 documents being returned.
-      **/
-      if( lastToken && ( lastToken.length > 1 || lastToken.match(/[0-9]/) ) ){
         clean.tokens_incomplete = [ lastToken ];
       }
     }

test/unit/fixture/autocomplete_linguistic_final_token.js

@@ -26,12 +26,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'one',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -48,12 +43,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'one',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -9,7 +9,8 @@ module.exports = {
             'boost': 100,
             'query': 'test',
             'type': 'phrase',
-            'operator': 'and'
+            'operator': 'and',
+            'slop': 3
           }
         }
       }],
@@ -22,7 +23,8 @@ module.exports = {
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
               }
             }
           },
@@ -41,7 +43,7 @@ module.exports = {
             'weight': 40
           }],
           'score_mode': 'avg',
-          'boost_mode': 'multiply',
+          'boost_mode': 'replace',
           'filter': {
             'or': [
               {
@@ -60,12 +62,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -82,12 +79,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -9,7 +9,8 @@ module.exports = {
             'boost': 100,
             'query': 'test',
             'type': 'phrase',
-            'operator': 'and'
+            'operator': 'and',
+            'slop': 3
           }
         }
       }],
@@ -22,7 +23,8 @@ module.exports = {
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
               }
             }
           },
@@ -41,7 +43,7 @@ module.exports = {
             'weight': 40
           }],
           'score_mode': 'avg',
-          'boost_mode': 'multiply',
+          'boost_mode': 'replace',
           'filter': {
             'or': [
               {
@@ -60,12 +62,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -82,12 +79,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_linguistic_multiple_tokens.js

@@ -20,7 +20,8 @@ module.exports = {
             'boost': 100,
             'query': 'three',
             'type': 'phrase',
-            'operator': 'and'
+            'operator': 'and',
+            'slop': 3
           }
         }
       }],
@@ -39,12 +40,7 @@ module.exports = {
         {
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'one two three',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -61,12 +57,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'one two three',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_linguistic_only.js

@@ -9,19 +9,15 @@ module.exports = {
             'boost': 100,
             'query': 'test',
             'type': 'phrase',
-            'operator': 'and'
+            'operator': 'and',
+            'slop': 3
           }
         }
       }],
       'should':[{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -38,12 +34,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_linguistic_with_admin.js

@@ -102,12 +102,7 @@ module.exports = {
         {
           'function_score': {
             'query': {
-              'match': {
+              'match_all': {}
-                'name.default': {
-                  'analyzer': 'peliasQueryFullToken',
-                  'query': 'one two',
-                }
-              }
             },
             'max_boost': 20,
             'functions': [
@@ -127,12 +122,7 @@ module.exports = {
         {
           'function_score': {
             'query': {
-              'match': {
+              'match_all': {}
-                'name.default': {
-                  'analyzer': 'peliasQueryFullToken',
-                  'query': 'one two',
-                }
-              }
             },
             'max_boost': 20,
             'functions': [

test/unit/fixture/autocomplete_single_character_street.js

@@ -101,12 +101,7 @@ module.exports = {
         {
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'k road',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -123,12 +118,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'k road',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/fixture/autocomplete_with_source_filtering.js

@@ -9,19 +9,15 @@ module.exports = {
             'boost': 100,
             'query': 'test',
             'type': 'phrase',
-            'operator': 'and'
+            'operator': 'and',
+            'slop': 3
           }
         }
       }],
       'should':[{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -38,12 +34,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-            'match': {
+            'match_all': {}
-              'name.default': {
-                'analyzer': 'peliasQueryFullToken',
-                'query': 'test',
-              }
-            }
           },
           'max_boost': 20,
           'score_mode': 'first',

test/unit/sanitiser/_tokenizer.js

@@ -372,8 +372,10 @@ module.exports.tests.final_token_single_gram = function(test, common) {
       'grolmanstrasse',
     ], 'tokens produced');
 
-    // last token removed!
+    // last token marked as 'incomplete'
-    t.deepEquals(clean.tokens_incomplete, [], 'no tokens');
+    t.deepEquals(clean.tokens_incomplete, [
+      'a'
+    ], 'tokens produced');
 
     // no errors/warnings produced
     t.deepEquals(messages.errors, [], 'no errors');