Merge pull request #577 from pelias/staging

Merge staging into production

helper/type_mapping.js

@@ -47,7 +47,8 @@ var SOURCE_MAPPING = addStandardTargetsToAliases(SOURCES, SOURCE_ALIASES);
 var LAYERS_BY_SOURCE = {
  openstreetmap: [ 'address', 'venue' ],
  openaddresses: [ 'address' ],
- geonames: [ 'country', 'region', 'county', 'locality', 'venue' ],
+ geonames: [ 'country','macroregion', 'region', 'county','localadmin',
+  'locality', 'neighbourhood', 'venue' ],
  whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region',
    'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
    'neighbourhood', 'microhood', 'disputed']

middleware/dedupe.js

@@ -51,9 +51,12 @@ function isDifferent(item1, item2) {
 
     if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
       for (var lang in item1.name) {
+        if(item2.name[lang] || lang === 'default') {
+          // do not consider absence of an additional name as a difference
           propMatch(item1.name, item2.name, lang);
         }
       }
+    }
     else {
       propMatch(item1, item2, 'name');
     }

package.json

@@ -49,10 +49,10 @@
     "lodash": "^4.5.0",
     "markdown": "0.5.0",
     "morgan": "1.7.0",
-    "pelias-config": "^1.0.1",
+    "pelias-config": "~2.0.0",
     "pelias-logger": "^0.0.8",
     "pelias-model": "^4.0.0",
-    "pelias-query": "7.0.1",
+    "pelias-query": "~8.0.0",
     "pelias-suggester-pipeline": "2.0.4",
     "pelias-text-analyzer": "^1.0.1",
     "stats-lite": "1.0.3",
@@ -66,8 +66,10 @@
     "nsp": "^2.2.0",
     "precommit-hook": "^3.0.0",
     "proxyquire": "^1.7.7",
+    "source-map": "^0.5.6",
     "tap-dot": "1.0.5",
-    "tape": "^4.5.1"
+    "tape": "^4.5.1",
+    "uglify-js": "^2.6.2"
   },
   "pre-commit": [
     "lint",

query/autocomplete_defaults.js

@@ -15,10 +15,8 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'boundary:circle:radius': '50km',
   'boundary:circle:distance_type': 'plane',
   'boundary:circle:optimize_bbox': 'indexed',
-  'boundary:circle:_cache': true,
 
   'boundary:rect:type': 'indexed',
-  'boundary:rect:_cache': true,
 
   'ngram:analyzer': 'peliasQueryPartialToken',
   'ngram:field': 'name.default',
@@ -36,7 +34,7 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'focus:weight': 40,
 
   'function_score:score_mode': 'avg',
-  'function_score:boost_mode': 'multiply',
+  'function_score:boost_mode': 'replace',
 
   'address:housenumber:analyzer': 'peliasHousenumber',
   'address:housenumber:field': 'address_parts.number',

query/reverse_defaults.js

@@ -16,10 +16,8 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'boundary:circle:radius:coarse': '500km',
   'boundary:circle:distance_type': 'plane',
   'boundary:circle:optimize_bbox': 'indexed',
-  'boundary:circle:_cache': true,
 
   'boundary:rect:type': 'indexed',
-  'boundary:rect:_cache': true,
 
   'ngram:analyzer': 'peliasQueryPartialToken',
   'ngram:field': 'name.default',
@@ -31,7 +29,7 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'phrase:slop': 2,
 
   'focus:function': 'linear',
-  'focus:offset': '1km',
+  'focus:offset': '0km',
   'focus:scale': '50km',
   'focus:decay': 0.5,
   'focus:weight': 2,

query/search_defaults.js

@@ -15,12 +15,10 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'boundary:circle:radius': '50km',
   'boundary:circle:distance_type': 'plane',
   'boundary:circle:optimize_bbox': 'indexed',
-  'boundary:circle:_cache': true,
 
   'boundary:rect:type': 'indexed',
-  'boundary:rect:_cache': true,
 
-  'ngram:analyzer': 'peliasIndexOneEdgeGram',
+  'ngram:analyzer': 'peliasQueryFullToken',
   'ngram:field': 'name.default',
   'ngram:boost': 1,
 
@@ -30,7 +28,7 @@ module.exports = _.merge({}, peliasQuery.defaults, {
   'phrase:slop': 2,
 
   'focus:function': 'linear',
-  'focus:offset': '1km',
+  'focus:offset': '0km',
   'focus:scale': '50km',
   'focus:decay': 0.5,
   'focus:weight': 2,

query/view/focus_selected_layers.js

@@ -14,6 +14,12 @@ var peliasQuery = require('pelias-query');
 module.exports = function( subview ){
   return function( vs ){
 
+    // don't perform this query on single character inputs
+    // as its too unperformant to sort a large part of the index.
+    if( vs.var('input:name').get().length < 2 ){
+      return null;
+    }
+
     if( !subview ){ return null; } // subview validation failed
     var macroView = peliasQuery.view.focus( subview );
     if( !macroView ){ return null; } // macroView validation failed

query/view/ngrams_last_token_only.js

@@ -28,5 +28,9 @@ module.exports = function( vs ){
   vsCopy.var('input:name').set( tokens.join(' ') );
 
   // return the view rendered using the copy
-  return ngrams_strict( vsCopy );
+  return {
+    'constant_score': {
+      'query': ngrams_strict( vsCopy )
+    }
+  };
 };

query/view/ngrams_strict.js

@@ -10,10 +10,16 @@ var peliasQuery = require('pelias-query');
 
 module.exports = function( vs ){
 
+  // validate required params
+  if( !vs.isset('phrase:slop') ){
+    return null;
+  }
+
   var view = peliasQuery.view.ngrams( vs );
 
   view.match['name.default'].type = 'phrase';
   view.match['name.default'].operator = 'and';
+  view.match['name.default'].slop = vs.var('phrase:slop');
 
   return view;
 };

query/view/pop_subquery.js

@@ -4,14 +4,13 @@ var peliasQuery = require('pelias-query'),
 
 /**
   Population / Popularity subquery
-**/
-
-module.exports = function( vs ){
 
-  var view = peliasQuery.view.ngrams( vs );
+  In prior versions we have had restricted the population/popularity boost
+  to only a section of the query results.
 
-  view.match['name.default'].analyzer = vs.var('phrase:analyzer');
+  Currently it is configured to `match_all`, ie. targets all records.
-  delete view.match['name.default'].boost;
+**/
 
-  return view;
+module.exports = function( vs ){
+  return { 'match_all': {} };
 };

sanitiser/_tokenizer.js

@@ -92,13 +92,7 @@ function sanitize( raw, clean ){
       // set all but the last token as 'complete'
       clean.tokens_complete = tokensCopy;
 
-      /**
+      if( lastToken ){
-        if the last token is a single non-numeric character then we must discard it.
-
-        at time of writing, single non-numeric ngrams are not stored in the index,
-        sending them as part of the query would result in 0 documents being returned.
-      **/
-      if( lastToken && ( lastToken.length > 1 || lastToken.match(/[0-9]/) ) ){
         clean.tokens_incomplete = [ lastToken ];
       }
     }

test/unit/fixture/autocomplete_linguistic_final_token.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -28,12 +26,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'one',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -50,12 +43,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'one',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -71,8 +59,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -1,17 +1,20 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
+        'constant_score': {
+          'query': {
             'match': {
               'name.default': {
                 'analyzer': 'peliasQueryPartialToken',
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
+              }
+            }
           }
         }
       }],
@@ -24,7 +27,8 @@ module.exports = {
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                    'operator': 'and'
+                'operator': 'and',
+                'slop': 3
               }
             }
           },
@@ -43,7 +47,7 @@ module.exports = {
             'weight': 40
           }],
           'score_mode': 'avg',
-              'boost_mode': 'multiply',
+          'boost_mode': 'replace',
           'filter': {
             'or': [
               {
@@ -62,12 +66,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -84,12 +83,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -105,8 +99,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -1,17 +1,20 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
+        'constant_score': {
+          'query': {
             'match': {
               'name.default': {
                 'analyzer': 'peliasQueryPartialToken',
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
+              }
+            }
           }
         }
       }],
@@ -24,7 +27,8 @@ module.exports = {
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                    'operator': 'and'
+                'operator': 'and',
+                'slop': 3
               }
             }
           },
@@ -43,7 +47,7 @@ module.exports = {
             'weight': 40
           }],
           'score_mode': 'avg',
-              'boost_mode': 'multiply',
+          'boost_mode': 'replace',
           'filter': {
             'or': [
               {
@@ -62,12 +66,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -84,12 +83,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -105,8 +99,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_linguistic_multiple_tokens.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -16,13 +14,18 @@ module.exports = {
         }
       },
       {
+        'constant_score': {
+          'query': {
             'match': {
               'name.default': {
                 'analyzer': 'peliasQueryPartialToken',
                 'boost': 100,
                 'query': 'three',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
+              }
+            }
           }
         }
       }],
@@ -41,12 +44,7 @@ module.exports = {
         {
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'one two three',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -63,12 +61,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'one two three',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -84,8 +77,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_linguistic_only.js

@@ -1,29 +1,27 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
+        'constant_score': {
+          'query': {
             'match': {
               'name.default': {
                 'analyzer': 'peliasQueryPartialToken',
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
+              }
+            }
           }
         }
       }],
       'should':[{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -40,12 +38,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -61,8 +54,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_linguistic_with_admin.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [
@@ -104,12 +102,7 @@ module.exports = {
         {
           'function_score': {
             'query': {
-                  'match': {
+              'match_all': {}
-                    'name.default': {
-                      'analyzer': 'peliasQueryFullToken',
-                      'query': 'one two',
-                    }
-                  }
             },
             'max_boost': 20,
             'functions': [
@@ -129,12 +122,7 @@ module.exports = {
         {
           'function_score': {
             'query': {
-                  'match': {
+              'match_all': {}
-                    'name.default': {
-                      'analyzer': 'peliasQueryFullToken',
-                      'query': 'one two',
-                    }
-                  }
             },
             'max_boost': 20,
             'functions': [
@@ -153,8 +141,6 @@ module.exports = {
         }
       ]
     }
-      }
-    }
   },
   'size': 20,
   'track_scores': true,

test/unit/fixture/autocomplete_single_character_street.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -103,12 +101,7 @@ module.exports = {
         {
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'k road',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -125,12 +118,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'k road',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -146,8 +134,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/autocomplete_with_source_filtering.js

@@ -1,29 +1,27 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
+        'constant_score': {
+          'query': {
             'match': {
               'name.default': {
                 'analyzer': 'peliasQueryPartialToken',
                 'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
-                'operator': 'and'
+                'operator': 'and',
+                'slop': 3
+              }
+            }
           }
         }
       }],
       'should':[{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -40,12 +38,7 @@ module.exports = {
       },{
         'function_score': {
           'query': {
-                'match': {
+            'match_all': {}
-                  'name.default': {
-                    'analyzer': 'peliasQueryFullToken',
-                    'query': 'test',
-                  }
-                }
           },
           'max_boost': 20,
           'score_mode': 'first',
@@ -59,19 +52,13 @@ module.exports = {
             'weight': 3
           }]
         }
-          }]
+      }],
-        }
+      'filter': [{
-      },
-      'filter': {
-        'bool': {
-          'must': [{
         'terms': {
           'source': ['test_source']
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/fixture/dedupe_elasticsearch_results.js

@@ -80,6 +80,34 @@ module.exports = [
     '_score': 1.2367082,
     'confidence': 0.879
   },
+  { // same as #1, but with an additional name
+    'center_point': {
+      'lon': -76.207456,
+      'lat': 40.039265
+    },
+    'address_parts': {},
+    'parent': {
+      'localadmin': ['East Lampeter'],
+      'region_a': ['PA'],
+      'region': ['Pennsylvania'],
+      'locality': ['Smoketown'],
+      'country_a': ['USA'],
+      'county': ['Lancaster County'],
+      'country': ['United States'],
+      'neighbourhood': ['Greenland']
+    },
+    'name': {
+      'default': 'East Lampeter High School',
+      'alt': 'High School of East Lampeter',
+    },
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'venue',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
   {
     'center_point': {
       'lon': -76.207456,

test/unit/fixture/reverse_boundary_circle.js

@@ -14,7 +14,6 @@ module.exports = {
                 'distance': vs.distance,
                 'distance_type': 'plane',
                 'optimize_bbox': 'indexed',
-                '_cache': true,
                 'center_point': {
                   'lat': 29.49136,
                   'lon': -82.50622

test/unit/fixture/reverse_null_island.js

@@ -2,30 +2,18 @@ var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
-    'filtered': {
-      'query': {
-        'bool': {
-          'must': []
-        }
-      },
-      'filter': {
     'bool': {
-          'must': [
+      'filter': [{
-            {
         'geo_distance': {
           'distance': '500km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
-                '_cache': true,
           'center_point': {
             'lat': 0,
             'lon': 0
           }
         }
-            }
+      }]
-          ]
-        }
-      }
     }
   },
   'sort': [

test/unit/fixture/reverse_standard.js

@@ -2,30 +2,18 @@ var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
-    'filtered': {
-      'query': {
-        'bool': {
-          'must': []
-        }
-      },
-      'filter': {
     'bool': {
-          'must': [
+      'filter': [{
-            {
         'geo_distance': {
           'distance': '500km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
-                '_cache': true,
           'center_point': {
             'lat': 29.49136,
             'lon': -82.50622
           }
         }
-            }
+      }]
-          ]
-        }
-      }
     }
   },
   'sort': [

test/unit/fixture/reverse_with_boundary_country.js

@@ -1,8 +1,6 @@
 var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [
@@ -14,27 +12,18 @@ module.exports = {
             }
           }
         }
-          ]
+      ],
-        }
+      'filter': [{
-      },
-      'filter': {
-        'bool': {
-          'must': [
-            {
         'geo_distance': {
           'distance': '500km',
           'distance_type': 'plane',
           'optimize_bbox': 'indexed',
-                '_cache': true,
           'center_point': {
             'lat': 29.49136,
             'lon': -82.50622
           }
         }
-            }
+      }]
-          ]
-        }
-      }
     }
   },
   'sort': [

test/unit/fixture/reverse_with_source_filtering.js

@@ -2,21 +2,13 @@ var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
-    'filtered': {
-      'query': {
-        'bool': {
-          'must': []
-        }
-      },
-      'filter': {
     'bool': {
-          'must': [
+      'filter': [
         {
           'geo_distance': {
             'distance': '500km',
             'distance_type': 'plane',
             'optimize_bbox': 'indexed',
-                '_cache': true,
             'center_point': {
               'lat': 29.49136,
               'lon': -82.50622
@@ -30,8 +22,6 @@ module.exports = {
         }
       ]
     }
-      }
-    }
   },
   'sort': [
     '_score',

test/unit/fixture/search_boundary_country.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [
@@ -18,7 +16,7 @@ module.exports = {
             'name.default': {
               'query': 'test',
               'boost': 1,
-                  'analyzer': 'peliasIndexOneEdgeGram'
+              'analyzer': 'peliasQueryFullToken'
             }
           }
         }
@@ -85,8 +83,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_full_address.js

@@ -1,15 +1,13 @@
 var vs = require('../../../query/search_defaults');
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
         'match': {
           'name.default': {
             'query': '123 main st',
-                'analyzer': 'peliasIndexOneEdgeGram',
+            'analyzer': 'peliasQueryFullToken',
             'boost': 1
           }
         }
@@ -132,8 +130,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'size': 10,
   'sort': [ '_score' ],

test/unit/fixture/search_linguistic_bbox.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -73,12 +71,8 @@ module.exports = {
             'weight': 2
           }]
         }
-          }]
+      }],
-        }
+      'filter': [{
-      },
-      'filter': {
-        'bool': {
-          'must': [{
         'geo_bounding_box': {
           'center_point': {
             'top': 11.51,
@@ -86,13 +80,10 @@ module.exports = {
             'bottom': 47.47,
             'left': -103.16
           },
-              '_cache': true,
           'type': 'indexed'
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_linguistic_focus.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -43,7 +41,7 @@ module.exports = {
                   'lat': 29.49136,
                   'lon': -82.50622
                 },
-                    'offset': '1km',
+                'offset': '0km',
                 'scale': '50km',
                 'decay': 0.5
               }
@@ -105,8 +103,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_linguistic_focus_bbox.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -43,7 +41,7 @@ module.exports = {
                   'lat': 29.49136,
                   'lon': -82.50622
                 },
-                    'offset': '1km',
+                'offset': '0km',
                 'scale': '50km',
                 'decay': 0.5
               }
@@ -103,12 +101,8 @@ module.exports = {
             'weight': 2
           }]
         }
-          }]
+      }],
-        }
+      'filter': [{
-      },
-      'filter': {
-        'bool': {
-          'must': [{
         'geo_bounding_box': {
           'center_point': {
             'top': 11.51,
@@ -116,13 +110,10 @@ module.exports = {
             'bottom': 47.47,
             'left': -103.16
           },
-              '_cache': true,
           'type': 'indexed'
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_linguistic_focus_null_island.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -43,7 +41,7 @@ module.exports = {
                   'lat': 0,
                   'lon': 0
                 },
-                    'offset': '1km',
+                'offset': '0km',
                 'scale': '50km',
                 'decay': 0.5
               }
@@ -105,8 +103,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_linguistic_only.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -75,8 +73,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 10,

test/unit/fixture/search_linguistic_viewport.js

@@ -1,13 +1,11 @@
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [
         {
           'match': {
             'name.default': {
-                  'analyzer': 'peliasIndexOneEdgeGram',
+              'analyzer': 'peliasQueryFullToken',
               'boost': 1,
               'query': 'test'
             }
@@ -48,7 +46,7 @@ module.exports = {
                       'lat': 29.49136,
                       'lon': -82.50622
                     },
-                        'offset': '1km',
+                    'offset': '0km',
                     'scale': '50km',
                     'decay': 0.5
                   }
@@ -117,8 +115,6 @@ module.exports = {
         }
       ]
     }
-      }
-    }
   },
   'size': 10,
   'track_scores': true,

test/unit/fixture/search_linguistic_viewport_min_diagonal.js

@@ -7,7 +7,7 @@ module.exports = {
             {
               'match': {
                 'name.default': {
-                  'analyzer': 'peliasIndexOneEdgeGram',
+                  'analyzer': 'peliasQueryFullToken',
                   'boost': 1,
                   'query': 'test'
                 }
@@ -48,7 +48,7 @@ module.exports = {
                           'lat': 28.49136,
                           'lon': -87.50623
                         },
-                        'offset': '1km',
+                        'offset': '0km',
                         'scale': '1km',
                         'decay': 0.5
                       }

test/unit/fixture/search_partial_address.js

@@ -2,15 +2,13 @@
 var vs = require('../../../query/search_defaults');
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
         'match': {
           'name.default': {
             'query': 'soho grand',
-                'analyzer': 'peliasIndexOneEdgeGram',
+            'analyzer': 'peliasQueryFullToken',
             'boost': 1
           }
         }
@@ -100,8 +98,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'size': 10,
   'sort': [ '_score' ],

test/unit/fixture/search_regions_address.js

@@ -2,15 +2,13 @@
 var vs = require('../../../query/search_defaults');
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
         'match': {
           'name.default': {
             'query': '1 water st',
-                'analyzer': 'peliasIndexOneEdgeGram',
+            'analyzer': 'peliasQueryFullToken',
             'boost': 1
           }
         }
@@ -116,8 +114,6 @@ module.exports = {
         }
       }]
     }
-      }
-    }
   },
   'size': 10,
   'sort': [ '_score' ],

test/unit/fixture/search_with_source_filtering.js

@@ -1,7 +1,5 @@
 
 module.exports = {
-  'query': {
-    'filtered': {
   'query': {
     'bool': {
       'must': [{
@@ -9,7 +7,7 @@ module.exports = {
           'name.default': {
             'query': 'test',
             'boost': 1,
-                'analyzer': 'peliasIndexOneEdgeGram'
+            'analyzer': 'peliasQueryFullToken'
           }
         }
       }],
@@ -73,19 +71,13 @@ module.exports = {
             'weight': 2
           }]
         }
-          }]
+      }],
-        }
+      'filter': [{
-      },
-      'filter': {
-        'bool': {
-          'must': [{
         'terms': {
           'source': ['test_source']
         }
       }]
     }
-      }
-    }
   },
   'sort': [ '_score' ],
   'size': 20,

test/unit/query/reverse.js

@@ -22,7 +22,7 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_standard');
 
-    t.deepEqual(compiled, expected, 'valid reverse query');
+    t.deepEqual(compiled, expected, 'reverse_standard');
     t.end();
   });
 
@@ -38,7 +38,7 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_null_island');
 
-    t.deepEqual(compiled, expected, 'valid reverse query');
+    t.deepEqual(compiled, expected, 'reverse_null_island');
     t.end();
   });
 
@@ -54,7 +54,7 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = '123km';
 
-    t.deepEqual(compiled.query.filtered.filter.bool.must[0].geo_distance.distance, expected, 'distance set to boundary circle radius');
+    t.deepEqual(compiled.query.bool.filter[0].geo_distance.distance, expected, 'distance set to boundary circle radius');
     t.end();
   });
 
@@ -71,9 +71,9 @@ module.exports.tests.query = function(test, common) {
 
     // this should not equal `point.lat` and `point.lon` as it was explitely specified
     var expected = { lat: clean['boundary.circle.lat'], lon: clean['boundary.circle.lon'] };
-    var centroid = compiled.query.filtered.filter.bool.must[0].geo_distance.center_point;
+    var centroid = compiled.query.bool.filter[0].geo_distance.center_point;
 
-    t.deepEqual(centroid, expected, 'boundary.circle/lon overrides point.lat/lon');
+    t.deepEqual(centroid, expected, 'reverse: boundary.circle/lon overrides point.lat/lon');
     t.end();
   });
 

test/unit/query/search.js

@@ -178,7 +178,7 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_boundary_country');
 
-    t.deepEqual(compiled, expected, 'valid boundary.country query');
+    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
     t.end();
   });
 
@@ -191,7 +191,7 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_source_filtering');
 
-    t.deepEqual(compiled, expected, 'valid search query with source filtering');
+    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
     t.end();
   });
 

test/unit/sanitiser/_sources_and_layers.js

@@ -51,6 +51,17 @@ module.exports.tests.no_errors = function(test, common) {
     t.end();
   });
 
+test('valid combination', function(t) {
+    var raw = {};
+    var clean = { sources: ['geonames'], layers: ['macroregion'] };
+
+    var messages = sanitize(raw, clean);
+
+    t.equal(messages.errors.length, 0, 'should return no errors');
+    t.equal(messages.warnings.length, 0, 'should return no warnings');
+    t.end();
+  });
+
   test('valid combination because of multiple sources', function(t) {
     var raw = {};
     var clean = { sources: ['openstreetmap', 'openaddresses'], layers: ['venue'] };

test/unit/sanitiser/_tokenizer.js

@@ -372,8 +372,10 @@ module.exports.tests.final_token_single_gram = function(test, common) {
       'grolmanstrasse',
     ], 'tokens produced');
 
-    // last token removed!
+    // last token marked as 'incomplete'
-    t.deepEquals(clean.tokens_incomplete, [], 'no tokens');
+    t.deepEquals(clean.tokens_incomplete, [
+      'a'
+    ], 'tokens produced');
 
     // no errors/warnings produced
     t.deepEquals(messages.errors, [], 'no errors');