From 908100d9c50e19387a09c053a1dc71610b7522a1 Mon Sep 17 00:00:00 2001
From: Julian Simioni <julian@mapzen.com>
Date: Wed, 24 Aug 2016 17:47:02 -0400
Subject: [PATCH 01/82] Explicitly set coarse alias rather than using WOF
 layers

Who's on First will have more than just coarse data now!
---
 helper/type_mapping.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/helper/type_mapping.js b/helper/type_mapping.js
index fde1fab1..8da5682c 100644
--- a/helper/type_mapping.js
+++ b/helper/type_mapping.js
@@ -60,7 +60,9 @@ var LAYERS_BY_SOURCE = {
  * may have layers that mean the same thing but have a different name
  */
 var LAYER_ALIASES = {
-  'coarse': LAYERS_BY_SOURCE.whosonfirst
+  'coarse': [ 'continent', 'country', 'dependency', 'macroregion', 'region',
+   'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
+   'neighbourhood', 'microhood', 'disputed']
 };
 
 // create a list of all layers by combining each entry from LAYERS_BY_SOURCE

From b0c0a5fe5cc541585d576186ce4f63038d462def Mon Sep 17 00:00:00 2001
From: Julian Simioni <julian@mapzen.com>
Date: Wed, 24 Aug 2016 17:48:16 -0400
Subject: [PATCH 02/82] Add venues to Who's on First

---
 helper/type_mapping.js                     |  2 +-
 test/unit/sanitiser/_sources_and_layers.js | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/helper/type_mapping.js b/helper/type_mapping.js
index 8da5682c..0d51ef81 100644
--- a/helper/type_mapping.js
+++ b/helper/type_mapping.js
@@ -51,7 +51,7 @@ var LAYERS_BY_SOURCE = {
   'locality','borough', 'neighbourhood', 'venue' ],
  whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region',
    'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough',
-   'neighbourhood', 'microhood', 'disputed']
+   'neighbourhood', 'microhood', 'disputed', 'venue']
 };
 
 /*
diff --git a/test/unit/sanitiser/_sources_and_layers.js b/test/unit/sanitiser/_sources_and_layers.js
index b943770c..5b8bf3a2 100644
--- a/test/unit/sanitiser/_sources_and_layers.js
+++ b/test/unit/sanitiser/_sources_and_layers.js
@@ -73,6 +73,17 @@ test('valid combination', function(t) {
     t.end();
   });
 
+  test('valid combination: wof venues', function(t) {
+    var raw = {};
+    var clean = { sources: ['whosonfirst'], layers: ['venue'] };
+
+    var messages = sanitize(raw, clean);
+
+    t.equal(messages.errors.length, 0, 'should return no errors');
+    t.equal(messages.warnings.length, 0, 'should return no warnings');
+    t.end();
+  });
+
   test('valid combination because of multiple sources', function(t) {
     var raw = {};
     var clean = { sources: ['openstreetmap', 'openaddresses'], layers: ['venue'] };

From 080ad1cbf6e63efc2f8b8359eea52521cb5bff7b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 20 May 2016 14:21:54 -0400
Subject: [PATCH 03/82] turned off multicore

---
 index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.js b/index.js
index 8df79add..d9503bf0 100644
--- a/index.js
+++ b/index.js
@@ -2,7 +2,7 @@
 var cluster = require('cluster'),
     app = require('./app'),
     port = ( process.env.PORT || 3100 ),
-    multicore = true;
+    multicore = false;
 
 /** cluster webserver across all cores **/
 if( multicore ){

From 246f9e7d435e4e64897caa0112f6d8391e05e4e9 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 20 May 2016 14:22:28 -0400
Subject: [PATCH 04/82] added generatePermutations to middleware

---
 routes/v1.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/routes/v1.js b/routes/v1.js
index 674c1f5c..468c5211 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -13,7 +13,8 @@ var sanitisers = {
 
 /** ----------------------- middleware ------------------------ **/
 var middleware = {
-  calcSize: require('../middleware/sizeCalculator')
+  calcSize: require('../middleware/sizeCalculator'),
+  generatePermutations: require('../middleware/generatePermutations')
 };
 
 /** ----------------------- controllers ----------------------- **/

From abef8f1efac91412b07a82425ad1728a4d40c314 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 20 May 2016 14:23:17 -0400
Subject: [PATCH 05/82] initial stab at generatePermutations

---
 middleware/generatePermutations.js           |  49 +++++
 test/unit/middleware/generatePermutations.js | 207 +++++++++++++++++++
 2 files changed, 256 insertions(+)
 create mode 100644 middleware/generatePermutations.js
 create mode 100644 test/unit/middleware/generatePermutations.js

diff --git a/middleware/generatePermutations.js b/middleware/generatePermutations.js
new file mode 100644
index 00000000..da4b56de
--- /dev/null
+++ b/middleware/generatePermutations.js
@@ -0,0 +1,49 @@
+var _ = require('lodash');
+
+function setup() {
+  return function generatePermutations(req, res, next) {
+    if (_.isUndefined(req.clean) || _.isEmpty(req.clean.parsed_text)) {
+      return next();
+    }
+
+    req.clean.permutations = [];
+
+    if (req.clean.parsed_text.hasOwnProperty('number')) {
+      req.clean.permutations.push(req.clean.parsed_text);
+
+      if (req.clean.parsed_text.hasOwnProperty('street')) {
+        req.clean.permutations.push({
+          street: req.clean.parsed_text.street,
+          city: req.clean.parsed_text.city,
+          state: req.clean.parsed_text.state
+        });
+
+      }
+
+      if (req.clean.parsed_text.hasOwnProperty('city')) {
+        req.clean.permutations.push({
+          city: req.clean.parsed_text.city,
+          state: req.clean.parsed_text.state
+        });
+
+      }
+
+      if (req.clean.parsed_text.hasOwnProperty('state')) {
+        req.clean.permutations.push({
+          state: req.clean.parsed_text.state
+        });
+      }
+
+    }
+
+    // { number: '102',
+    //   street: 'south charles st',
+    //   city: 'red lion',
+    //   state: 'pa' }
+
+    next();
+  };
+
+}
+
+module.exports = setup;
diff --git a/test/unit/middleware/generatePermutations.js b/test/unit/middleware/generatePermutations.js
new file mode 100644
index 00000000..c05f30dd
--- /dev/null
+++ b/test/unit/middleware/generatePermutations.js
@@ -0,0 +1,207 @@
+var generatePermutations = require('../../../middleware/generatePermutations')();
+
+module.exports.tests = {};
+
+module.exports.tests.confidenceScore = function(test, common) {
+  test('undefined req.clean should not throw exception', function(t) {
+    var req = {};
+    var res = {};
+    var next_called = false;
+
+    function testIt() {
+      generatePermutations(req, res, function() { next_called = true; });
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.ok(next_called);
+    t.end();
+
+  });
+
+  test('undefined req.clean.parsed_text should not throw exception', function(t) {
+    var req = {
+      clean: {}
+    };
+    var res = {};
+    var next_called = false;
+
+    function testIt() {
+      generatePermutations(req, res, function() { next_called = true; });
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.ok(next_called);
+    t.equal(req.clean.permutations, undefined);
+    t.end();
+
+  });
+
+  test('empty req.clean.parsed_text should not throw exception', function(t) {
+    var req = {
+      clean: {
+        parsed_text: {}
+      }
+    };
+    var res = {};
+    var next_called = false;
+
+    function testIt() {
+      generatePermutations(req, res, function() { next_called = true; });
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.ok(next_called);
+    t.equal(req.clean.permutations, undefined);
+    t.end();
+
+  });
+
+  test('parsed_text with number should add permutations for less granular searches', function(t) {
+    var req = {
+      clean: {
+        parsed_text: {
+          number: '1234',
+          street: 'street name',
+          city: 'city name',
+          state: 'state name'
+        }
+      }
+    };
+    var res = {};
+    var next_called = false;
+
+    function testIt() {
+      generatePermutations(req, res, function() { next_called = true; });
+    }
+
+    var expected_permutations = [
+      {
+        number: '1234',
+        street: 'street name',
+        city: 'city name',
+        state: 'state name'
+      },
+      {
+        street: 'street name',
+        city: 'city name',
+        state: 'state name'
+      },
+      {
+        city: 'city name',
+        state: 'state name'
+      },
+      {
+        state: 'state name'
+      }
+    ];
+
+    testIt();
+
+    t.deepEquals(req.clean.permutations, expected_permutations);
+    t.ok(next_called);
+    t.end();
+
+  });
+
+  //
+  // test('empty res and req should not throw exception', function(t) {
+  //   function testIt() {
+  //     confidenceScore({}, {}, function() {});
+  //   }
+  //
+  //   t.doesNotThrow(testIt, 'an exception should not have been thrown');
+  //   t.end();
+  // });
+  //
+  // test('res.results without parsed_text should not throw exception', function(t) {
+  //   var req = {};
+  //   var res = {
+  //     data: [{
+  //       name: 'foo'
+  //     }],
+  //     meta: [10]
+  //   };
+  //
+  //   function testIt() {
+  //     confidenceScore(req, res, function() {});
+  //   }
+  //
+  //   t.doesNotThrow(testIt, 'an exception should not have been thrown');
+  //   t.end();
+  // });
+  //
+  // test('hit without address should not error', function(t) {
+  //   var req = {
+  //     clean: {
+  //       text: 'test name3',
+  //       parsed_text: {
+  //         postalcode: 12345
+  //       }
+  //     }
+  //   };
+  //   var res = {
+  //     data: [{
+  //       name: {
+  //         default: 'foo'
+  //       }
+  //     }],
+  //     meta: {
+  //       scores: [10]
+  //     }
+  //   };
+  //
+  //   function testIt() {
+  //     confidenceScore(req, res, function() {});
+  //   }
+  //
+  //   t.doesNotThrow(testIt, 'an exception should not have been thrown with no address');
+  //   t.end();
+  // });
+  //
+  //
+  // test('res.results without parsed_text should not throw exception', function(t) {
+  //   var req = {
+  //     clean: { text: 'test name1' }
+  //   };
+  //   var res = {
+  //     data: [{
+  //       _score: 10,
+  //       found: true,
+  //       value: 1,
+  //       center_point: { lat: 100.1, lon: -50.5 },
+  //       name: { default: 'test name1' },
+  //       parent: {
+  //         country: ['country1'],
+  //         region: ['state1'],
+  //         county: ['city1']
+  //       }
+  //     }, {
+  //       _score: 20,
+  //       value: 2,
+  //       center_point: { lat: 100.2, lon: -51.5 },
+  //       name: { default: 'test name2' },
+  //       parent: {
+  //         country: ['country2'],
+  //         region: ['state2'],
+  //         county: ['city2']
+  //       }
+  //     }],
+  //     meta: {scores: [10]}
+  //   };
+  //
+  //   confidenceScore(req, res, function() {});
+  //   t.equal(res.data[0].confidence, 0.6, 'score was set');
+  //   t.end();
+  // });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('[middleware] confidenceScore: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From b1cf06720beac072033c30fe541bac06af07556e Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 20 May 2016 14:24:02 -0400
Subject: [PATCH 06/82] included generatePermutations in list of tests

---
 test/unit/run.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/run.js b/test/unit/run.js
index de8d770a..ae9790c6 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -61,7 +61,7 @@ var tests = [
   require('./sanitiser/search'),
   require('./sanitiser/wrap'),
   require('./service/mget'),
-  require('./service/search'),
+  require('./service/search')
 ];
 
 tests.map(function(t) {

From ab9b1dbafab62843e30810ce39daedefe4199013 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 20 May 2016 14:24:31 -0400
Subject: [PATCH 07/82] commented out deprecated AddressIt code

---
 query/text_parser.js | 72 ++++++++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/query/text_parser.js b/query/text_parser.js
index 00e60724..4f3df8b6 100644
--- a/query/text_parser.js
+++ b/query/text_parser.js
@@ -19,22 +19,22 @@ var adminFields = placeTypes.concat([
 function addParsedVariablesToQueryVariables( parsed_text, vs ){
 
   // is it a street address?
-  var isStreetAddress = parsed_text.hasOwnProperty('number') && parsed_text.hasOwnProperty('street');
-  if( isStreetAddress ){
-    vs.var( 'input:name', parsed_text.number + ' ' + parsed_text.street );
-  }
-
-  // ?
-  else if( parsed_text.admin_parts ) {
-    vs.var( 'input:name', parsed_text.name );
-  }
-
-  // ?
-  else {
-    logger.warn( 'chaos monkey asks: what happens now?' );
-    logger.warn( parsed_text );
-    try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
-  }
+  // var isStreetAddress = parsed_text.hasOwnProperty('number') && parsed_text.hasOwnProperty('street');
+  // if( isStreetAddress ){
+  //   vs.var( 'input:name', parsed_text.number + ' ' + parsed_text.street );
+  // }
+  //
+  // // ?
+  // else if( parsed_text.admin_parts ) {
+  //   vs.var( 'input:name', parsed_text.name );
+  // }
+  //
+  // // ?
+  // else {
+  //   logger.warn( 'chaos monkey asks: what happens now?' );
+  //   logger.warn( parsed_text );
+  //   try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
+  // }
 
   // ==== add parsed matches [address components] ====
 
@@ -74,26 +74,26 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
   // @todo: clean up this code
 
   // a concept called 'leftovers' which is just 'admin_parts' /or 'regions'.
-  var leftoversString = '';
-  if( parsed_text.hasOwnProperty('admin_parts') ){
-    leftoversString = parsed_text.admin_parts;
-  }
-  else if( parsed_text.hasOwnProperty('regions') ){
-    leftoversString = parsed_text.regions.join(' ');
-  }
-
-  // if we have 'leftovers' then assign them to any fields which
-  // currently don't have a value assigned.
-  if( leftoversString.length ){
-
-    // cycle through fields and set fields which
-    // are still currently unset
-    adminFields.forEach( function( key ){
-      if( !vs.isset( 'input:' + key ) ){
-        vs.var( 'input:' + key, leftoversString );
-      }
-    });
-  }
+  // var leftoversString = '';
+  // if( parsed_text.hasOwnProperty('admin_parts') ){
+  //   leftoversString = parsed_text.admin_parts;
+  // }
+  // else if( parsed_text.hasOwnProperty('regions') ){
+  //   leftoversString = parsed_text.regions.join(' ');
+  // }
+  //
+  // // if we have 'leftovers' then assign them to any fields which
+  // // currently don't have a value assigned.
+  // if( leftoversString.length ){
+  //
+  //   // cycle through fields and set fields which
+  //   // are still currently unset
+  //   adminFields.forEach( function( key ){
+  //     if( !vs.isset( 'input:' + key ) ){
+  //       vs.var( 'input:' + key, leftoversString );
+  //     }
+  //   });
+  // }
 }
 
 module.exports = addParsedVariablesToQueryVariables;

From 0166940e1ad9ec1db7a8bf17214b1d1206b2bbd7 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Tue, 31 May 2016 15:10:11 -0400
Subject: [PATCH 08/82] added debug

---
 query/search.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/query/search.js b/query/search.js
index 5ab96248..f1d86cc7 100644
--- a/query/search.js
+++ b/query/search.js
@@ -125,7 +125,10 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  return query.render( vs );
+  var q = query.render(vs);
+  console.log(JSON.stringify(q, null, 2));
+
+  return q;
 }
 
 module.exports = generateQuery;

From c7e5ba2e2a1ecab4c0e2b422ca30b89b7c0c5550 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 16 Jun 2016 11:17:22 -0400
Subject: [PATCH 09/82] added postprocessing step to trim by granularity

---
 middleware/trimByGranularity.js           |  42 ++++
 test/unit/middleware/trimByGranularity.js | 254 ++++++++++++++++++++++
 2 files changed, 296 insertions(+)
 create mode 100644 middleware/trimByGranularity.js
 create mode 100644 test/unit/middleware/trimByGranularity.js

diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js
new file mode 100644
index 00000000..2579585b
--- /dev/null
+++ b/middleware/trimByGranularity.js
@@ -0,0 +1,42 @@
+var _ = require('lodash');
+
+// layers in increasing order of granularity
+var layers = [
+  ['venue'],
+  ['address'],
+  ['neighbourhood'],
+  ['locality', 'localadmin'],
+  ['county', 'macrocounty'],
+  ['region', 'macroregion'],
+  ['country']
+];
+
+function hasRecordsAtLayers(results, layers) {
+  return _.some(results, function(result) {
+    return layers.indexOf(result.layer) !== -1;
+  });
+}
+
+function retainRecordsAtLayers(results, layers) {
+  return _.filter(results, function(result) {
+    return layers.indexOf(result.layer) !== -1;
+  });
+}
+
+function setup() {
+ return function trim(req, res, next) {
+   if (_.isUndefined(req.clean)) {
+     return next();
+   }
+
+   layers.forEach(function(layer) {
+     if (hasRecordsAtLayers(res.data, layer )) {
+       res.data = retainRecordsAtLayers(res.data, layer);
+     }
+   });
+
+   next();
+ };
+}
+
+module.exports = setup;
diff --git a/test/unit/middleware/trimByGranularity.js b/test/unit/middleware/trimByGranularity.js
new file mode 100644
index 00000000..0d23cb74
--- /dev/null
+++ b/test/unit/middleware/trimByGranularity.js
@@ -0,0 +1,254 @@
+var trimByGranularity = require('../../../middleware/trimByGranularity')();
+
+module.exports.tests = {};
+
+module.exports.tests.trimByGranularity = function(test, common) {
+  test('empty res and req should not throw exception', function(t) {
+    function testIt() {
+      trimByGranularity({}, {}, function() {});
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.end();
+  });
+
+  test('when venue records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'venue 1', layer: 'venue' },
+        { name: 'venue 2', layer: 'venue' },
+        { name: 'address 1', layer: 'address' },
+        { name: 'neighbourhood 1', layer: 'neighbourhood' },
+        { name: 'locality 1', layer: 'locality' },
+        { name: 'localadmin 1', layer: 'localadmin' },
+        { name: 'county 1', layer: 'county' },
+        { name: 'macrocounty 1', layer: 'macrocounty' },
+        { name: 'region 1', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'venue 1', layer: 'venue' },
+      { name: 'venue 2', layer: 'venue' },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only venue records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when address records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'address 1', layer: 'address' },
+        { name: 'address 2', layer: 'address' },
+        { name: 'neighbourhood 1', layer: 'neighbourhood' },
+        { name: 'locality 1', layer: 'locality' },
+        { name: 'localadmin 1', layer: 'localadmin' },
+        { name: 'county 1', layer: 'county' },
+        { name: 'macrocounty 1', layer: 'macrocounty' },
+        { name: 'region 1', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' },
+      ]
+    };
+
+    var expected_data = [
+      { name: 'address 1', layer: 'address' },
+      { name: 'address 2', layer: 'address' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only address records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when neighbourhood records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'neighbourhood 1', layer: 'neighbourhood' },
+        { name: 'neighbourhood 2', layer: 'neighbourhood' },
+        { name: 'locality 1', layer: 'locality' },
+        { name: 'locality 2', layer: 'locality' },
+        { name: 'localadmin 1', layer: 'localadmin' },
+        { name: 'localadmin 2', layer: 'localadmin' },
+        { name: 'county 1', layer: 'county' },
+        { name: 'macrocounty 1', layer: 'macrocounty' },
+        { name: 'region 1', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'neighbourhood 1', layer: 'neighbourhood' },
+      { name: 'neighbourhood 2', layer: 'neighbourhood' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only neighbourhood records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when locality/localadmin records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'locality 1', layer: 'locality' },
+        { name: 'locality 2', layer: 'locality' },
+        { name: 'localadmin 1', layer: 'localadmin' },
+        { name: 'localadmin 2', layer: 'localadmin' },
+        { name: 'county 1', layer: 'county' },
+        { name: 'macrocounty 1', layer: 'macrocounty' },
+        { name: 'region 1', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' },
+      ]
+    };
+
+    var expected_data = [
+      { name: 'locality 1', layer: 'locality' },
+      { name: 'locality 2', layer: 'locality' },
+      { name: 'localadmin 1', layer: 'localadmin' },
+      { name: 'localadmin 2', layer: 'localadmin' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only locality/localadmin records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when county/macrocounty records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'county 1', layer: 'county' },
+        { name: 'county 2', layer: 'county' },
+        { name: 'macrocounty 1', layer: 'macrocounty' },
+        { name: 'macrocounty 2', layer: 'macrocounty' },
+        { name: 'region 1', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' },
+      ]
+    };
+
+    var expected_data = [
+      { name: 'county 1', layer: 'county' },
+      { name: 'county 2', layer: 'county' },
+      { name: 'macrocounty 1', layer: 'macrocounty' },
+      { name: 'macrocounty 2', layer: 'macrocounty' },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only county/macrocounty records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when region/macroregion records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'region 1', layer: 'region' },
+        { name: 'region 2', layer: 'region' },
+        { name: 'macroregion 1', layer: 'macroregion' },
+        { name: 'macroregion 2', layer: 'macroregion' },
+        { name: 'country 1', layer: 'country' },
+      ]
+    };
+
+    var expected_data = [
+      { name: 'region 1', layer: 'region' },
+      { name: 'region 2', layer: 'region' },
+      { name: 'macroregion 1', layer: 'macroregion' },
+      { name: 'macroregion 2', layer: 'macroregion' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only region/macroregion records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+  test('when country records are most granular, only they should be retained', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'country 1', layer: 'country' },
+        { name: 'country 2', layer: 'country' }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'country 1', layer: 'country' },
+      { name: 'country 2', layer: 'country' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only country records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('[middleware] trimByGranularity: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From 5f259d420b173f6fd620ba45ce947dfd18b3cbf1 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 27 Jul 2016 21:31:53 -0400
Subject: [PATCH 10/82] updated text-analyzer version, removed unused module

---
 middleware/generatePermutations.js           |  49 -----
 test/unit/middleware/generatePermutations.js | 207 -------------------
 2 files changed, 256 deletions(-)
 delete mode 100644 middleware/generatePermutations.js
 delete mode 100644 test/unit/middleware/generatePermutations.js

diff --git a/middleware/generatePermutations.js b/middleware/generatePermutations.js
deleted file mode 100644
index da4b56de..00000000
--- a/middleware/generatePermutations.js
+++ /dev/null
@@ -1,49 +0,0 @@
-var _ = require('lodash');
-
-function setup() {
-  return function generatePermutations(req, res, next) {
-    if (_.isUndefined(req.clean) || _.isEmpty(req.clean.parsed_text)) {
-      return next();
-    }
-
-    req.clean.permutations = [];
-
-    if (req.clean.parsed_text.hasOwnProperty('number')) {
-      req.clean.permutations.push(req.clean.parsed_text);
-
-      if (req.clean.parsed_text.hasOwnProperty('street')) {
-        req.clean.permutations.push({
-          street: req.clean.parsed_text.street,
-          city: req.clean.parsed_text.city,
-          state: req.clean.parsed_text.state
-        });
-
-      }
-
-      if (req.clean.parsed_text.hasOwnProperty('city')) {
-        req.clean.permutations.push({
-          city: req.clean.parsed_text.city,
-          state: req.clean.parsed_text.state
-        });
-
-      }
-
-      if (req.clean.parsed_text.hasOwnProperty('state')) {
-        req.clean.permutations.push({
-          state: req.clean.parsed_text.state
-        });
-      }
-
-    }
-
-    // { number: '102',
-    //   street: 'south charles st',
-    //   city: 'red lion',
-    //   state: 'pa' }
-
-    next();
-  };
-
-}
-
-module.exports = setup;
diff --git a/test/unit/middleware/generatePermutations.js b/test/unit/middleware/generatePermutations.js
deleted file mode 100644
index c05f30dd..00000000
--- a/test/unit/middleware/generatePermutations.js
+++ /dev/null
@@ -1,207 +0,0 @@
-var generatePermutations = require('../../../middleware/generatePermutations')();
-
-module.exports.tests = {};
-
-module.exports.tests.confidenceScore = function(test, common) {
-  test('undefined req.clean should not throw exception', function(t) {
-    var req = {};
-    var res = {};
-    var next_called = false;
-
-    function testIt() {
-      generatePermutations(req, res, function() { next_called = true; });
-    }
-
-    t.doesNotThrow(testIt, 'an exception should not have been thrown');
-    t.ok(next_called);
-    t.end();
-
-  });
-
-  test('undefined req.clean.parsed_text should not throw exception', function(t) {
-    var req = {
-      clean: {}
-    };
-    var res = {};
-    var next_called = false;
-
-    function testIt() {
-      generatePermutations(req, res, function() { next_called = true; });
-    }
-
-    t.doesNotThrow(testIt, 'an exception should not have been thrown');
-    t.ok(next_called);
-    t.equal(req.clean.permutations, undefined);
-    t.end();
-
-  });
-
-  test('empty req.clean.parsed_text should not throw exception', function(t) {
-    var req = {
-      clean: {
-        parsed_text: {}
-      }
-    };
-    var res = {};
-    var next_called = false;
-
-    function testIt() {
-      generatePermutations(req, res, function() { next_called = true; });
-    }
-
-    t.doesNotThrow(testIt, 'an exception should not have been thrown');
-    t.ok(next_called);
-    t.equal(req.clean.permutations, undefined);
-    t.end();
-
-  });
-
-  test('parsed_text with number should add permutations for less granular searches', function(t) {
-    var req = {
-      clean: {
-        parsed_text: {
-          number: '1234',
-          street: 'street name',
-          city: 'city name',
-          state: 'state name'
-        }
-      }
-    };
-    var res = {};
-    var next_called = false;
-
-    function testIt() {
-      generatePermutations(req, res, function() { next_called = true; });
-    }
-
-    var expected_permutations = [
-      {
-        number: '1234',
-        street: 'street name',
-        city: 'city name',
-        state: 'state name'
-      },
-      {
-        street: 'street name',
-        city: 'city name',
-        state: 'state name'
-      },
-      {
-        city: 'city name',
-        state: 'state name'
-      },
-      {
-        state: 'state name'
-      }
-    ];
-
-    testIt();
-
-    t.deepEquals(req.clean.permutations, expected_permutations);
-    t.ok(next_called);
-    t.end();
-
-  });
-
-  //
-  // test('empty res and req should not throw exception', function(t) {
-  //   function testIt() {
-  //     confidenceScore({}, {}, function() {});
-  //   }
-  //
-  //   t.doesNotThrow(testIt, 'an exception should not have been thrown');
-  //   t.end();
-  // });
-  //
-  // test('res.results without parsed_text should not throw exception', function(t) {
-  //   var req = {};
-  //   var res = {
-  //     data: [{
-  //       name: 'foo'
-  //     }],
-  //     meta: [10]
-  //   };
-  //
-  //   function testIt() {
-  //     confidenceScore(req, res, function() {});
-  //   }
-  //
-  //   t.doesNotThrow(testIt, 'an exception should not have been thrown');
-  //   t.end();
-  // });
-  //
-  // test('hit without address should not error', function(t) {
-  //   var req = {
-  //     clean: {
-  //       text: 'test name3',
-  //       parsed_text: {
-  //         postalcode: 12345
-  //       }
-  //     }
-  //   };
-  //   var res = {
-  //     data: [{
-  //       name: {
-  //         default: 'foo'
-  //       }
-  //     }],
-  //     meta: {
-  //       scores: [10]
-  //     }
-  //   };
-  //
-  //   function testIt() {
-  //     confidenceScore(req, res, function() {});
-  //   }
-  //
-  //   t.doesNotThrow(testIt, 'an exception should not have been thrown with no address');
-  //   t.end();
-  // });
-  //
-  //
-  // test('res.results without parsed_text should not throw exception', function(t) {
-  //   var req = {
-  //     clean: { text: 'test name1' }
-  //   };
-  //   var res = {
-  //     data: [{
-  //       _score: 10,
-  //       found: true,
-  //       value: 1,
-  //       center_point: { lat: 100.1, lon: -50.5 },
-  //       name: { default: 'test name1' },
-  //       parent: {
-  //         country: ['country1'],
-  //         region: ['state1'],
-  //         county: ['city1']
-  //       }
-  //     }, {
-  //       _score: 20,
-  //       value: 2,
-  //       center_point: { lat: 100.2, lon: -51.5 },
-  //       name: { default: 'test name2' },
-  //       parent: {
-  //         country: ['country2'],
-  //         region: ['state2'],
-  //         county: ['city2']
-  //       }
-  //     }],
-  //     meta: {scores: [10]}
-  //   };
-  //
-  //   confidenceScore(req, res, function() {});
-  //   t.equal(res.data[0].confidence, 0.6, 'score was set');
-  //   t.end();
-  // });
-
-};
-
-module.exports.all = function (tape, common) {
-  function test(name, testFunction) {
-    return tape('[middleware] confidenceScore: ' + name, testFunction);
-  }
-
-  for( var testCase in module.exports.tests ){
-    module.exports.tests[testCase](test, common);
-  }
-};

From 8d532adb3fdf897cea2bc9f87e0658afca208325 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 29 Jul 2016 12:59:16 -0400
Subject: [PATCH 11/82] added support for `_matched_queries` in ES hits

this will make it easier to sort thru results from FallbackQuery by knowing which query was called
---
 service/search.js           | 1 +
 test/unit/mock/backend.js   | 2 ++
 test/unit/service/search.js | 8 +++++---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/service/search.js b/service/search.js
index c5aad5a9..fe04d88f 100644
--- a/service/search.js
+++ b/service/search.js
@@ -35,6 +35,7 @@ function service( backend, cmd, cb ){
         hit._source._id = hit._id;
         hit._source._type = hit._type;
         hit._source._score = hit._score;
+        hit._source._matched_queries = hit._matched_queries;
 
         return hit._source;
       });
diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js
index 7d347a30..7f288246 100644
--- a/test/unit/mock/backend.js
+++ b/test/unit/mock/backend.js
@@ -11,6 +11,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){
     _id: 'myid1',
     _type: 'mytype1',
     _score: 10,
+    _matched_queries: ['query 1', 'query 2'],
     _source: {
       value: 1,
       center_point: { lat: 100.1, lon: -50.5 },
@@ -21,6 +22,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){
     _id: 'myid2',
     _type: 'mytype2',
     _score: 20,
+    _matched_queries: ['query 3'],
     _source: {
       value: 2,
       center_point: { lat: 100.2, lon: -51.5 },
diff --git a/test/unit/service/search.js b/test/unit/service/search.js
index 0de8e0fa..cf72b75e 100644
--- a/test/unit/service/search.js
+++ b/test/unit/service/search.js
@@ -13,21 +13,23 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-// functionally test service 
+// functionally test service
 module.exports.tests.functional_success = function(test, common) {
 
   var expected = [
     {
       _id: 'myid1', _type: 'mytype1',
       _score: 10,
+      _matched_queries: ['query 1', 'query 2'],
       value: 1,
       center_point: { lat: 100.1, lon: -50.5 },
       name: { default: 'test name1' },
       parent: { country: ['country1'], region: ['state1'], county: ['city1'] }
-    }, 
+    },
     {
       _id: 'myid2', _type: 'mytype2',
       _score: 20,
+      _matched_queries: ['query 3'],
       value: 2,
       center_point: { lat: 100.2, lon: -51.5 },
       name: { default: 'test name2' },
@@ -88,4 +90,4 @@ module.exports.all = function (tape, common) {
   for( var testCase in module.exports.tests ){
     module.exports.tests[testCase](test, common);
   }
-};
\ No newline at end of file
+};

From c18baaf77512b9ba1520b9ca6378f711552fe67b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 29 Jul 2016 15:24:58 -0400
Subject: [PATCH 12/82] fixed text analysis tests

proxyquire is now used because the text_analyzer package requires node_postal which isn't guaranteed to be available
---
 sanitiser/_text.js           |   5 +-
 test/unit/sanitiser/_text.js | 130 ++++++++++++++++++++++++++++++++++-
 2 files changed, 130 insertions(+), 5 deletions(-)

diff --git a/sanitiser/_text.js b/sanitiser/_text.js
index 4709eeee..874a9b17 100644
--- a/sanitiser/_text.js
+++ b/sanitiser/_text.js
@@ -1,5 +1,5 @@
 var check = require('check-types'),
-    text_analyzer = require('pelias-text-analyzer');
+  text_analyzer = require('pelias-text-analyzer');
 
 // validate texts, convert types and apply defaults
 function sanitize( raw, clean ){
@@ -8,13 +8,14 @@ function sanitize( raw, clean ){
   var messages = { errors: [], warnings: [] };
 
   // invalid input 'text'
+  // must call `!check.nonEmptyString` since `check.emptyString` returns
+  //  `false` for `undefined` and `null`
   if( !check.nonEmptyString( raw.text ) ){
     messages.errors.push('invalid param \'text\': text length, must be >0');
   }
 
   // valid input 'text'
   else {
-
     // valid text
     clean.text = raw.text;
 
diff --git a/test/unit/sanitiser/_text.js b/test/unit/sanitiser/_text.js
index 3868c86c..82dfb5e0 100644
--- a/test/unit/sanitiser/_text.js
+++ b/test/unit/sanitiser/_text.js
@@ -1,23 +1,147 @@
-var sanitiser = require('../../../sanitiser/_text');
 var type_mapping = require('../../../helper/type_mapping');
+var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
 
 module.exports.tests.text_parser = function(test, common) {
-  test('short input text has admin layers set ', function(t) {
+  test('non-empty raw.text should call analyzer and set clean.text and clean.parsed_text', function(t) {
+    var mock_analyzer_response = {
+      key1: 'value 1',
+      key2: 'value 2'
+    };
+
+    var sanitiser = proxyquire('../../../sanitiser/_text', {
+      'pelias-text-analyzer': { parse: function(query) {
+        return mock_analyzer_response;
+      }
+    }});
+
+    var raw = {
+      text: 'raw input'
+    };
+    var clean = {
+    };
+
+    var expected_clean = {
+      text: raw.text,
+      parsed_text: mock_analyzer_response
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('empty raw.text should add error message', function(t) {
+    var sanitiser = proxyquire('../../../sanitiser/_text', {
+      'pelias-text-analyzer': { parse: function(query) {
+        throw new Error('analyzer should not have been called');
+      }
+    }});
+
+    var raw = {
+      text: ''
+    };
+    var clean = {
+    };
+
+    var expected_clean = {
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('undefined raw.text should add error message', function(t) {
+    var sanitiser = proxyquire('../../../sanitiser/_text', {
+      'pelias-text-analyzer': { parse: function(query) {
+        throw new Error('analyzer should not have been called');
+      }
+    }});
+
     var raw = {
-      text: 'emp'  //start of empire state building
+      text: undefined
     };
     var clean = {
     };
 
+    var expected_clean = {
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('text_analyzer.parse returning undefined should not overwrite clean.parsed_text', function(t) {
+    var sanitiser = proxyquire('../../../sanitiser/_text', {
+      'pelias-text-analyzer': { parse: function(query) {
+        return undefined;
+      }
+    }});
+
+    var raw = {
+      text: 'raw input'
+    };
+    var clean = {
+      parsed_text: 'original clean.parsed_text'
+    };
+
+    var expected_clean = {
+      text: raw.text,
+      parsed_text: 'original clean.parsed_text'
+    };
+
     var messages = sanitiser(raw, clean);
 
+    t.deepEquals(clean, expected_clean);
     t.deepEquals(messages.errors, [], 'no errors');
     t.deepEquals(messages.warnings, [], 'no warnings');
+    t.end();
+
+  });
+
+  test('text_analyzer.parse returning null should not overwrite clean.parsed_text', function(t) {
+    var sanitiser = proxyquire('../../../sanitiser/_text', {
+      'pelias-text-analyzer': { parse: function(query) {
+        return null;
+      }
+    }});
+
+    var raw = {
+      text: 'raw input'
+    };
+    var clean = {
+      parsed_text: 'original clean.parsed_text'
+    };
 
+    var expected_clean = {
+      text: raw.text,
+      parsed_text: 'original clean.parsed_text'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(clean, expected_clean);
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
     t.end();
+
   });
+
 };
 
 module.exports.all = function (tape, common) {

From 5821ae6e81bb6ce2c31bfa4c871506293acb8d0b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 29 Jul 2016 16:20:52 -0400
Subject: [PATCH 13/82] libpostal changes

- updated for libpostal interface
- added tests for text_parser
---
 query/text_parser.js           | 87 +++++++++++-----------------------
 test/unit/query/text_parser.js | 78 ++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+), 60 deletions(-)
 create mode 100644 test/unit/query/text_parser.js

diff --git a/query/text_parser.js b/query/text_parser.js
index 4f3df8b6..c71e444a 100644
--- a/query/text_parser.js
+++ b/query/text_parser.js
@@ -1,42 +1,18 @@
-
 var logger = require('pelias-logger').get('api');
-var placeTypes = require('../helper/placeTypes');
-
-/*
-This list should only contain admin fields we are comfortable matching in the case
-when we can't identify parts of an address. This shouldn't contain fields like country_a
-or postalcode because we should only try to match those when we're sure that's what they are.
- */
-var adminFields = placeTypes.concat([
-  'region_a'
-]);
-
-/**
-  @todo: refactor me
-**/
 
 // all the address parsing logic
 function addParsedVariablesToQueryVariables( parsed_text, vs ){
+  // ==== add parsed matches [address components] ====
 
-  // is it a street address?
-  // var isStreetAddress = parsed_text.hasOwnProperty('number') && parsed_text.hasOwnProperty('street');
-  // if( isStreetAddress ){
-  //   vs.var( 'input:name', parsed_text.number + ' ' + parsed_text.street );
-  // }
-  //
-  // // ?
-  // else if( parsed_text.admin_parts ) {
-  //   vs.var( 'input:name', parsed_text.name );
-  // }
-  //
-  // // ?
-  // else {
-  //   logger.warn( 'chaos monkey asks: what happens now?' );
-  //   logger.warn( parsed_text );
-  //   try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
-  // }
+  // query - Mexitaly, Sunoco, Lowes
+  if (parsed_text.hasOwnProperty('query')) {
+    vs.var('input:query', parsed_text.query);
+  }
 
-  // ==== add parsed matches [address components] ====
+  // categories - restaurants, hotels, bars
+  if (parsed_text.hasOwnProperty('category')) {
+    vs.var('input:category', parsed_text.category);
+  }
 
   // house number
   if( parsed_text.hasOwnProperty('number') ){
@@ -48,6 +24,16 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
     vs.var( 'input:street', parsed_text.street );
   }
 
+  // neighbourhood
+  if (parsed_text.hasOwnProperty('neighbourhood')) {
+    vs.var( 'input:neighbourhood', parsed_text.neighbourhood);
+  }
+
+  // borough
+  if (parsed_text.hasOwnProperty('borough')) {
+    vs.var( 'input:borough', parsed_text.borough);
+  }
+
   // postal code
   if( parsed_text.hasOwnProperty('postalcode') ){
     vs.var( 'input:postcode', parsed_text.postalcode );
@@ -57,43 +43,24 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
 
   // city
   if( parsed_text.hasOwnProperty('city') ){
-    vs.var( 'input:county', parsed_text.city );
+    vs.var( 'input:locality', parsed_text.city );
+  }
+
+  // county
+  if( parsed_text.hasOwnProperty('county') ){
+    vs.var( 'input:county', parsed_text.county );
   }
 
   // state
   if( parsed_text.hasOwnProperty('state') ){
-    vs.var( 'input:region_a', parsed_text.state );
+    vs.var( 'input:region', parsed_text.state );
   }
 
   // country
   if( parsed_text.hasOwnProperty('country') ){
-    vs.var( 'input:country_a', parsed_text.country );
+    vs.var( 'input:country', parsed_text.country );
   }
 
-  // ==== deal with the 'leftover' components ====
-  // @todo: clean up this code
-
-  // a concept called 'leftovers' which is just 'admin_parts' /or 'regions'.
-  // var leftoversString = '';
-  // if( parsed_text.hasOwnProperty('admin_parts') ){
-  //   leftoversString = parsed_text.admin_parts;
-  // }
-  // else if( parsed_text.hasOwnProperty('regions') ){
-  //   leftoversString = parsed_text.regions.join(' ');
-  // }
-  //
-  // // if we have 'leftovers' then assign them to any fields which
-  // // currently don't have a value assigned.
-  // if( leftoversString.length ){
-  //
-  //   // cycle through fields and set fields which
-  //   // are still currently unset
-  //   adminFields.forEach( function( key ){
-  //     if( !vs.isset( 'input:' + key ) ){
-  //       vs.var( 'input:' + key, leftoversString );
-  //     }
-  //   });
-  // }
 }
 
 module.exports = addParsedVariablesToQueryVariables;
diff --git a/test/unit/query/text_parser.js b/test/unit/query/text_parser.js
new file mode 100644
index 00000000..5ffc5b76
--- /dev/null
+++ b/test/unit/query/text_parser.js
@@ -0,0 +1,78 @@
+var VariableStore = require('pelias-query').Vars;
+var text_parser = require('../../../query/text_parser');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof text_parser, 'function', 'valid function');
+    t.end();
+  });
+};
+
+module.exports.tests.query = function(test, common) {
+  test('parsed_text without properties should leave vs properties unset', function(t) {
+    var parsed_text = {};
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.false(vs.isset('input:query'));
+    t.false(vs.isset('input:category'));
+    t.false(vs.isset('input:housenumber'));
+    t.false(vs.isset('input:street'));
+    t.false(vs.isset('input:neighbourhood'));
+    t.false(vs.isset('input:borough'));
+    t.false(vs.isset('input:postcode'));
+    t.false(vs.isset('input:locality'));
+    t.false(vs.isset('input:county'));
+    t.false(vs.isset('input:region'));
+    t.false(vs.isset('input:country'));
+    t.end();
+
+  });
+
+  test('parsed_text without properties should leave vs properties unset', function(t) {
+    var parsed_text = {
+      query: 'query value',
+      category: 'category value',
+      number: 'number value',
+      street: 'street value',
+      neighbourhood: 'neighbourhood value',
+      borough: 'borough value',
+      postalcode: 'postalcode value',
+      city: 'city value',
+      county: 'county value',
+      state: 'state value',
+      country: 'country value'
+    };
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.equals(vs.var('input:query').toString(), 'query value');
+    t.equals(vs.var('input:category').toString(), 'category value');
+    t.equals(vs.var('input:housenumber').toString(), 'number value');
+    t.equals(vs.var('input:street').toString(), 'street value');
+    t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value');
+    t.equals(vs.var('input:borough').toString(), 'borough value');
+    t.equals(vs.var('input:postcode').toString(), 'postalcode value');
+    t.equals(vs.var('input:locality').toString(), 'city value');
+    t.equals(vs.var('input:county').toString(), 'county value');
+    t.equals(vs.var('input:region').toString(), 'state value');
+    t.equals(vs.var('input:country').toString(), 'country value');
+    t.end();
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('text_parser ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From 65f5cef9bf06a841a7d79dd72e74ff29a76edc39 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 29 Jul 2016 16:28:50 -0400
Subject: [PATCH 14/82] updated proxyquire version

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 31c8d36f..dc099a12 100644
--- a/package.json
+++ b/package.json
@@ -66,7 +66,7 @@
     "jshint": "^2.5.6",
     "nsp": "^2.2.0",
     "precommit-hook": "^3.0.0",
-    "proxyquire": "^1.7.7",
+    "proxyquire": "^1.7.10",
     "source-map": "^0.5.6",
     "tap-dot": "1.0.5",
     "tape": "^4.5.1",

From d3febe49cce0c4c6b8701111dbfd6420e524081d Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 1 Aug 2016 15:42:14 -0400
Subject: [PATCH 15/82] moved location of `matched_queries` to correct place

---
 service/search.js         | 2 +-
 test/unit/mock/backend.js | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/service/search.js b/service/search.js
index fe04d88f..780da2ae 100644
--- a/service/search.js
+++ b/service/search.js
@@ -35,7 +35,7 @@ function service( backend, cmd, cb ){
         hit._source._id = hit._id;
         hit._source._type = hit._type;
         hit._source._score = hit._score;
-        hit._source._matched_queries = hit._matched_queries;
+        hit._source._matched_queries = hit.matched_queries;
 
         return hit._source;
       });
diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js
index 7f288246..739ed2cb 100644
--- a/test/unit/mock/backend.js
+++ b/test/unit/mock/backend.js
@@ -11,7 +11,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){
     _id: 'myid1',
     _type: 'mytype1',
     _score: 10,
-    _matched_queries: ['query 1', 'query 2'],
+    matched_queries: ['query 1', 'query 2'],
     _source: {
       value: 1,
       center_point: { lat: 100.1, lon: -50.5 },
@@ -22,7 +22,7 @@ responses['client/search/ok/1'] = function( cmd, cb ){
     _id: 'myid2',
     _type: 'mytype2',
     _score: 20,
-    _matched_queries: ['query 3'],
+    matched_queries: ['query 3'],
     _source: {
       value: 2,
       center_point: { lat: 100.2, lon: -51.5 },

From 7495cad4653e9bd3226193eaf0ca7967aba2cff4 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 1 Aug 2016 16:46:42 -0400
Subject: [PATCH 16/82] added middleware component to trim by result
 granularity

purpose in comments, cleans up from overly-verbose FallbackQuery
---
 middleware/trimByGranularity.js           |  55 +++++--
 test/unit/middleware/trimByGranularity.js | 184 +++++++++++-----------
 2 files changed, 134 insertions(+), 105 deletions(-)

diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js
index 2579585b..9594e879 100644
--- a/middleware/trimByGranularity.js
+++ b/middleware/trimByGranularity.js
@@ -1,34 +1,61 @@
 var _ = require('lodash');
 
+// This middleware component trims the results array by granularity when
+// FallbackQuery was used.  FallbackQuery is used for inputs like
+// `1090 N Charlotte St, Lancaster, PA` where the address may not exist and
+// we must fall back to trying `Lancaster, PA`.  If the address does exist then
+// FallbackQuery will return results for:
+// - address+city+state
+// - city+state
+// - state
+//
+// Because the address matched, we're not interested in city+state or state, so
+// this component removes that aren't the most granular.  
+
 // layers in increasing order of granularity
 var layers = [
-  ['venue'],
-  ['address'],
-  ['neighbourhood'],
-  ['locality', 'localadmin'],
-  ['county', 'macrocounty'],
-  ['region', 'macroregion'],
-  ['country']
+  'venue',
+  'address',
+  'neighbourhood',
+  'borough',
+  'locality',
+  'county',
+  'region',
+  'country'
 ];
 
-function hasRecordsAtLayers(results, layers) {
-  return _.some(results, function(result) {
-    return layers.indexOf(result.layer) !== -1;
+// this helper method returns `true` if every result has a matched_query
+//  starting with `fallback.`
+function isFallbackQuery(results) {
+  return results.every(function(result) {
+    return result.hasOwnProperty('_matched_queries') &&
+            !_.isEmpty(result._matched_queries) &&
+            _.startsWith(result._matched_queries[0], 'fallback.');
+  });
+}
+
+function hasRecordsAtLayers(results, layer) {
+  return results.some(function(result) {
+    return result._matched_queries[0] === 'fallback.' + layer;
   });
 }
 
-function retainRecordsAtLayers(results, layers) {
-  return _.filter(results, function(result) {
-    return layers.indexOf(result.layer) !== -1;
+function retainRecordsAtLayers(results, layer) {
+  return results.filter(function(result) {
+    return result._matched_queries[0] === 'fallback.' + layer;
   });
 }
 
 function setup() {
  return function trim(req, res, next) {
-   if (_.isUndefined(req.clean)) {
+   // don't do anything if there are no results or there are non-fallback.* named queries
+   // there should never be a mixture of fallback.* and non-fallback.* named queries
+   if (_.isUndefined(res.data) || !isFallbackQuery(res.data)) {
      return next();
    }
 
+   // start at the most granular possible layer.  if there are results at a layer
+   // then remove everything not at that layer.
    layers.forEach(function(layer) {
      if (hasRecordsAtLayers(res.data, layer )) {
        res.data = retainRecordsAtLayers(res.data, layer);
diff --git a/test/unit/middleware/trimByGranularity.js b/test/unit/middleware/trimByGranularity.js
index 0d23cb74..07e6db9a 100644
--- a/test/unit/middleware/trimByGranularity.js
+++ b/test/unit/middleware/trimByGranularity.js
@@ -12,28 +12,26 @@ module.exports.tests.trimByGranularity = function(test, common) {
     t.end();
   });
 
-  test('when venue records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'venue 1', layer: 'venue' },
-        { name: 'venue 2', layer: 'venue' },
-        { name: 'address 1', layer: 'address' },
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' }
+        { name: 'venue 1', _matched_queries: ['fallback.venue'] },
+        { name: 'venue 2', _matched_queries: ['fallback.venue'] },
+        { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'venue 1', layer: 'venue' },
-      { name: 'venue 2', layer: 'venue' },
+      { name: 'venue 1', _matched_queries: ['fallback.venue'] },
+      { name: 'venue 2', _matched_queries: ['fallback.venue'] },
     ];
 
     function testIt() {
@@ -44,30 +42,27 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when address records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only addresses when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'address 1', layer: 'address' },
-        { name: 'address 2', layer: 'address' },
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'address 2', _matched_queries: ['fallback.address'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'address 1', layer: 'address' },
-      { name: 'address 2', layer: 'address' }
+      { name: 'address 1', _matched_queries: ['fallback.address'] },
+      { name: 'address 2', _matched_queries: ['fallback.address'] },
     ];
 
     function testIt() {
@@ -78,31 +73,26 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when neighbourhood records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only neighbourhoods when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'neighbourhood 2', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'locality 2', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'localadmin 2', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' }
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'neighbourhood 1', layer: 'neighbourhood' },
-      { name: 'neighbourhood 2', layer: 'neighbourhood' }
+      { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+      { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] },
     ];
 
     function testIt() {
@@ -113,121 +103,106 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when locality/localadmin records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only localities when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'locality 2', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'localadmin 2', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'locality 2', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'locality 1', layer: 'locality' },
-      { name: 'locality 2', layer: 'locality' },
-      { name: 'localadmin 1', layer: 'localadmin' },
-      { name: 'localadmin 2', layer: 'localadmin' }
+      { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+      { name: 'locality 2', _matched_queries: ['fallback.locality'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only locality/localadmin records should be here');
+        t.deepEquals(res.data, expected_data, 'only locality records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when county/macrocounty records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'county 1', layer: 'county' },
-        { name: 'county 2', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'macrocounty 2', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'county 2', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'county 1', layer: 'county' },
-      { name: 'county 2', layer: 'county' },
-      { name: 'macrocounty 1', layer: 'macrocounty' },
-      { name: 'macrocounty 2', layer: 'macrocounty' },
+      { name: 'county 1', _matched_queries: ['fallback.county'] },
+      { name: 'county 2', _matched_queries: ['fallback.county'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only county/macrocounty records should be here');
+        t.deepEquals(res.data, expected_data, 'only county records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when region/macroregion records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'region 1', layer: 'region' },
-        { name: 'region 2', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'macroregion 2', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'region 2', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'region 1', layer: 'region' },
-      { name: 'region 2', layer: 'region' },
-      { name: 'macroregion 1', layer: 'macroregion' },
-      { name: 'macroregion 2', layer: 'macroregion' }
+      { name: 'region 1', _matched_queries: ['fallback.region'] },
+      { name: 'region 2', _matched_queries: ['fallback.region'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only region/macroregion records should be here');
+        t.deepEquals(res.data, expected_data, 'only region records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when country records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only countries when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'country 1', layer: 'country' },
-        { name: 'country 2', layer: 'country' }
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'country 2', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'country 1', layer: 'country' },
-      { name: 'country 2', layer: 'country' }
+      { name: 'country 1', _matched_queries: ['fallback.country'] },
+      { name: 'country 2', _matched_queries: ['fallback.country'] },
     ];
 
     function testIt() {
@@ -237,6 +212,33 @@ module.exports.tests.trimByGranularity = function(test, common) {
       });
     }
 
+    testIt();
+  });
+
+  test('presence of any non-fallback.* named queries should not trim', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'region', _matched_queries: ['fallback.region'] },
+        { name: 'country', _matched_queries: ['fallback.country'] },
+        { name: 'result with non-named query' }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'region', _matched_queries: ['fallback.region'] },
+      { name: 'country', _matched_queries: ['fallback.country'] },
+      { name: 'result with non-named query' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'all should results should have been retained');
+        t.end();
+      });
+    }
+
     testIt();
 
   });

From 58a3a91793dbda2f58e7d5fa911e7ad475f9c23d Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Tue, 2 Aug 2016 11:59:41 -0400
Subject: [PATCH 17/82] added `trimByGranularity` post-processing step

---
 routes/v1.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/routes/v1.js b/routes/v1.js
index 468c5211..dd5ba0a9 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -13,8 +13,7 @@ var sanitisers = {
 
 /** ----------------------- middleware ------------------------ **/
 var middleware = {
-  calcSize: require('../middleware/sizeCalculator'),
-  generatePermutations: require('../middleware/generatePermutations')
+  calcSize: require('../middleware/sizeCalculator')
 };
 
 /** ----------------------- controllers ----------------------- **/
@@ -29,6 +28,7 @@ var controllers = {
 /** ----------------------- controllers ----------------------- **/
 
 var postProc = {
+  trimByGranularity: require('../middleware/trimByGranularity'),
   distances: require('../middleware/distance'),
   confidenceScores: require('../middleware/confidenceScore'),
   confidenceScoresReverse: require('../middleware/confidenceScoreReverse'),
@@ -64,6 +64,7 @@ function addRoutes(app, peliasConfig) {
       sanitisers.search.middleware,
       middleware.calcSize(),
       controllers.search(peliasConfig),
+      postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),

From 0aebf81ace9727f9bc8fbb30d70dc0d8c2c1c160 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Tue, 2 Aug 2016 12:11:54 -0400
Subject: [PATCH 18/82] support new querying model

add support for FallbackQuery and GeodisambiguationQuery from the pelias-query module.  GeodisambiguationQuery is used when the text-analyzer returns a single administrative area field.  FallbackQuery is used otherwise.
---
 query/search.js | 73 +++++++++++++++++++++++++++++++------------------
 1 file changed, 46 insertions(+), 27 deletions(-)

diff --git a/query/search.js b/query/search.js
index f1d86cc7..c7a26758 100644
--- a/query/search.js
+++ b/query/search.js
@@ -3,49 +3,47 @@ var peliasQuery = require('pelias-query'),
     textParser = require('./text_parser'),
     check = require('check-types');
 
-var placeTypes = require('../helper/placeTypes');
-
-// region_a is also an admin field. addressit tries to detect
-// region_a, in which case we use a match query specifically for it.
-// but address it doesn't know about all of them so it helps to search
-// against this with the other admin parts as a fallback
-var adminFields = placeTypes.concat(['region_a']);
-
 //------------------------------
 // general-purpose search query
 //------------------------------
-var query = new peliasQuery.layout.FilteredBooleanQuery();
+var fallbackQuery = new peliasQuery.layout.FallbackQuery();
+fallbackQuery.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
+fallbackQuery.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
+
+var geodisambiguationQuery = new peliasQuery.layout.GeodisambiguationQuery();
+geodisambiguationQuery.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
+geodisambiguationQuery.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
 
 // mandatory matches
-query.score( peliasQuery.view.boundary_country, 'must' );
-query.score( peliasQuery.view.ngrams, 'must' );
+// query.score( peliasQuery.view.boundary_country, 'must' );
+// query.score( peliasQuery.view.ngrams, 'must' );
 
 // scoring boost
-query.score( peliasQuery.view.phrase );
-query.score( peliasQuery.view.focus( peliasQuery.view.phrase ) );
-query.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
-query.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
+// query.score( peliasQuery.view.phrase );
+// query.score( peliasQuery.view.focus( peliasQuery.view.phrase ) );
+// query.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
+// query.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
 
 // address components
-query.score( peliasQuery.view.address('housenumber') );
-query.score( peliasQuery.view.address('street') );
-query.score( peliasQuery.view.address('postcode') );
+// query.score( peliasQuery.view.address('housenumber'), 'must' );
+// query.score( peliasQuery.view.address('street'), 'must' );
+// query.score( peliasQuery.view.address('postcode'), 'must' );
 
 // admin components
 // country_a and region_a are left as matches here because the text-analyzer
 // can sometimes detect them, in which case a query more specific than a
 // multi_match is appropriate.
-query.score( peliasQuery.view.admin('country_a') );
-query.score( peliasQuery.view.admin('region_a') );
-query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') );
+// query.score( peliasQuery.view.admin('country_a'), 'must' );
+// query.score( peliasQuery.view.admin('region_a'), 'must' );
+// query.score( peliasQuery.view.admin('locality'), 'must' );
+// query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') );
 
 // non-scoring hard filters
-query.filter( peliasQuery.view.boundary_circle );
-query.filter( peliasQuery.view.boundary_rect );
-query.filter( peliasQuery.view.sources );
-query.filter( peliasQuery.view.layers );
-query.filter( peliasQuery.view.categories );
-
+// query.filter( peliasQuery.view.boundary_circle );
+// query.filter( peliasQuery.view.boundary_rect );
+// query.filter( peliasQuery.view.sources );
+// query.filter( peliasQuery.view.layers );
+// query.filter( peliasQuery.view.categories );
 // --------------------------------
 
 /**
@@ -131,4 +129,25 @@ function generateQuery( clean ){
   return q;
 }
 
+function getQuery(vs) {
+  if (isSingleFieldGeoambiguity(vs) && !hasQueryOrAddress(vs)) {
+    return geodisambiguationQuery.render(vs);
+  } else {
+    return fallbackQuery.render(vs);
+  }
+
+}
+
+function isSingleFieldGeoambiguity(vs) {
+  return ['neighbourhood', 'borough', 'locality', 'county', 'region', 'country'].filter(function(layer) {
+    return vs.isset('input:' + layer);
+  }).length === 1;
+}
+
+function hasQueryOrAddress(vs) {
+  return ['housenumber', 'street', 'query', 'category'].filter(function(layer) {
+    return vs.isset('input:' + layer);
+  }).length > 0;
+}
+
 module.exports = generateQuery;

From f55aed2b1a97617229e432c3f7a4e057c4ad6f53 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 3 Aug 2016 14:31:21 -0400
Subject: [PATCH 19/82] finished master rebase, checking in all-but-tests
 commit

rebase ended up being more complicated than was expected so i'm breaking this into a few commits
---
 test/unit/controller/search.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 3e7494ac..3f74fbd7 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -54,6 +54,7 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid1',
       _score: 10,
       _type: 'mytype1',
+      _matched_queries: ['query 1', 'query 2'],
       parent: {
         country: ['country1'],
         region: ['state1'],
@@ -67,6 +68,7 @@ module.exports.tests.functional_success = function(test, common) {
       _id: 'myid2',
       _score: 20,
       _type: 'mytype2',
+      _matched_queries: ['query 3'],
       parent: {
         country: ['country2'],
         region: ['state2'],

From c257eb5fc3622e2e0c2f6c0665101a78db145040 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 3 Aug 2016 15:41:17 -0400
Subject: [PATCH 20/82] added tests for new queries

left in place but commented out tests that will be supported when boundary, focus, and sources/layers issues are resolved.
---
 test/unit/fixture/search_fallback.js          | 371 ++++++++++++++++++
 test/unit/fixture/search_geodisambiguation.js | 199 ++++++++++
 test/unit/query/search.js                     | 335 ++++++++--------
 3 files changed, 749 insertions(+), 156 deletions(-)
 create mode 100644 test/unit/fixture/search_fallback.js
 create mode 100644 test/unit/fixture/search_geodisambiguation.js

diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
new file mode 100644
index 00000000..304841b5
--- /dev/null
+++ b/test/unit/fixture/search_fallback.js
@@ -0,0 +1,371 @@
+module.exports = {
+  'query': {
+    'bool': {
+      'should': [
+        {
+          'bool': {
+            '_name': 'fallback.venue',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'query value',
+                  'type': 'phrase',
+                  'fields': [
+                    'phrase.default'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.neighbourhood',
+                    'parent.neighbourhood_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'borough value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.borough',
+                    'parent.borough_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'city value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a',
+                    'parent.localadmin',
+                    'parent.localadmin_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'venue'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.address',
+            'must': [
+              {
+                'match_phrase': {
+                  'address_parts.number': 'number value'
+                }
+              },
+              {
+                'match_phrase': {
+                  'address_parts.street': 'street value'
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.neighbourhood',
+                    'parent.neighbourhood_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'borough value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.borough',
+                    'parent.borough_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'city value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a',
+                    'parent.localadmin',
+                    'parent.localadmin_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'address'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.neighbourhood',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.neighbourhood',
+                    'parent.neighbourhood_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'borough value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.borough',
+                    'parent.borough_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'city value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a',
+                    'parent.localadmin',
+                    'parent.localadmin_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'neighbourhood'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.borough',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'borough value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.borough',
+                    'parent.borough_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'city value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a',
+                    'parent.localadmin',
+                    'parent.localadmin_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'borough'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.locality',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'city value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'locality'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.region',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'region'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            '_name': 'fallback.country',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'country'
+              }
+            }
+          }
+        }
+      ]
+    }
+  },
+  'size': 20,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_geodisambiguation.js b/test/unit/fixture/search_geodisambiguation.js
new file mode 100644
index 00000000..bd81e237
--- /dev/null
+++ b/test/unit/fixture/search_geodisambiguation.js
@@ -0,0 +1,199 @@
+module.exports = {
+  'query': {
+    'bool': {
+      'should': [
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.neighbourhood',
+                    'parent.neighbourhood_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'neighbourhood'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.borough',
+                    'parent.borough_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'borough'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.locality',
+                    'parent.locality_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'locality'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.localadmin',
+                    'parent.localadmin_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'localadmin'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'county'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'macrocounty'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'region'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.macroregion',
+                    'parent.macroregion_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'macroregion'
+              }
+            }
+          }
+        },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'country'
+              }
+            }
+          }
+        }
+      ]
+    }
+  },
+  'size': 20,
+  'track_scores': true
+};
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index 5abb35f7..2f7fa951 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -10,178 +10,201 @@ module.exports.tests.interface = function(test, common) {
 };
 
 module.exports.tests.query = function(test, common) {
-  test('valid search + focus + bbox', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
-      'boundary.rect.min_lat': 47.47,
-      'boundary.rect.max_lon': -61.84,
-      'boundary.rect.max_lat': 11.51,
-      'boundary.rect.min_lon': -103.16,
-      layers: ['test']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus_bbox');
-
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
-    t.end();
-  });
-
-  test('valid search + bbox', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      'boundary.rect.min_lat': 47.47,
-      'boundary.rect.max_lon': -61.84,
-      'boundary.rect.max_lat': 11.51,
-      'boundary.rect.min_lon': -103.16,
-      layers: ['test']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_bbox');
-
-    t.deepEqual(compiled, expected, 'search_linguistic_bbox');
-    t.end();
-  });
-
-  test('valid lingustic-only search', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      layers: ['test']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_only');
-
-    t.deepEqual(compiled, expected, 'search_linguistic_only');
-    t.end();
-  });
+  // test('valid search + focus + bbox', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+  //     'boundary.rect.min_lat': 47.47,
+  //     'boundary.rect.max_lon': -61.84,
+  //     'boundary.rect.max_lat': 11.51,
+  //     'boundary.rect.min_lon': -103.16,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_focus_bbox');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
+  //   t.end();
+  // });
+
+  // test('valid search + bbox', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'boundary.rect.min_lat': 47.47,
+  //     'boundary.rect.max_lon': -61.84,
+  //     'boundary.rect.max_lat': 11.51,
+  //     'boundary.rect.min_lon': -103.16,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_bbox');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_bbox');
+  //   t.end();
+  // });
+
+  // test('valid lingustic-only search', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_only');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_only');
+  //   t.end();
+  // });
+
+  // test('search search + focus', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_focus');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_focus');
+  //   t.end();
+  // });
+
+  // test('search search + viewport', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.viewport.min_lat': 28.49136,
+  //     'focus.viewport.max_lat': 30.49136,
+  //     'focus.viewport.min_lon': -87.50622,
+  //     'focus.viewport.max_lon': -77.50622,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_viewport');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_viewport');
+  //   t.end();
+  // });
+
+  // viewport scale sizing currently disabled.
+  // ref: https://github.com/pelias/api/pull/388
+  // test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.viewport.min_lat': 28.49135,
+  //     'focus.viewport.max_lat': 28.49137,
+  //     'focus.viewport.min_lon': -87.50622,
+  //     'focus.viewport.max_lon': -87.50624,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
+  //
+  //   t.deepEqual(compiled, expected, 'valid search query');
+  //   t.end();
+  // });
+
+  // test('search search + focus on null island', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.point.lat': 0, 'focus.point.lon': 0,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_focus_null_island');
+  //
+  //   t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
+  //   t.end();
+  // });
+
+  test('parsed_text with all fields should use FallbackQuery', function(t) {
+    var clean = {
+      parsed_text: {
+        query: 'query value',
+        category: 'category value',
+        number: 'number value',
+        street: 'street value',
+        neighbourhood: 'neighbourhood value',
+        borough: 'borough value',
+        postalcode: 'postalcode value',
+        city: 'city value',
+        county: 'county value',
+        state: 'state value',
+        country: 'country value'
+      }
+    };
 
-  test('search search + focus', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
-      layers: ['test']
-    });
+    var query = generate(clean);
 
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus');
+    var compiled = JSON.parse(JSON.stringify(query));
+    var expected = require('../fixture/search_fallback');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus');
+    t.deepEqual(compiled, expected, 'fallbackQuery');
     t.end();
-  });
 
-  test('search search + focus on null island', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      'focus.point.lat': 0, 'focus.point.lon': 0,
-      layers: ['test']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus_null_island');
-
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
-    t.end();
   });
 
-  test('valid query with a full valid address', function(t) {
-    var query = generate({ text: '123 main st new york ny 10010 US',
-      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-      querySize: 10,
+  test('parsed_text with single admin field should use GeodisambiguationQuery', function(t) {
+    var clean = {
       parsed_text: {
-        number: '123',
-        street: 'main st',
-        state: 'NY',
-        country: 'USA',
-        postalcode: '10010',
-        regions: [ 'new york' ]
+        neighbourhood: 'neighbourhood value'
       }
-    });
+    };
 
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_full_address');
+    var query = generate(clean);
 
-    t.deepEqual(compiled, expected, 'search_full_address');
-    t.end();
-  });
+    var compiled = JSON.parse(JSON.stringify(query));
+    var expected = require('../fixture/search_geodisambiguation');
 
-  test('valid query with partial address', function(t) {
-    var query = generate({ text: 'soho grand, new york',
-      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-      querySize: 10,
-      parsed_text: { name: 'soho grand',
-        state: 'NY',
-        regions: [ 'soho grand' ],
-        admin_parts: 'new york'
-      }
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_partial_address');
-
-    t.deepEqual(compiled, expected, 'search_partial_address');
-    t.end();
-  });
-
-  test('valid query with regions in address', function(t) {
-    var query = generate({ text: '1 water st manhattan ny',
-      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-      querySize: 10,
-      parsed_text: { number: '1',
-        street: 'water st',
-        state: 'NY',
-        regions: [ 'manhattan' ]
-      },
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_regions_address');
-
-    t.deepEqual(compiled, expected, 'search_regions_address');
+    t.deepEqual(compiled, expected, 'geodisambiguationQuery');
     t.end();
-  });
-
-  test('valid boundary.country search', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      layers: ['test'],
-      'boundary.country': 'ABC'
-    });
 
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_boundary_country');
-
-    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
-    t.end();
   });
 
-  test('valid sources filter', function(t) {
-    var query = generate({
-      'text': 'test',
-      'sources': ['test_source']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_with_source_filtering');
-
-    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
-    t.end();
-  });
-
-  test('categories filter', function(t) {
-    var query = generate({
-      'text': 'test',
-      'categories': ['retail','food']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_with_category_filtering');
-
-    t.deepEqual(compiled, expected, 'valid search query with category filtering');
-    t.end();
-  });
+  // test('valid boundary.country search', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     layers: ['test'],
+  //     'boundary.country': 'ABC'
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_boundary_country');
+  //
+  //   t.deepEqual(compiled, expected, 'search: valid boundary.country query');
+  //   t.end();
+  // });
+
+  // test('valid sources filter', function(t) {
+  //   var query = generate({
+  //     'text': 'test',
+  //     'sources': ['test_source']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_with_source_filtering');
+  //
+  //   t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
+  //   t.end();
+  // });
+
+  //test('categories filter', function(t) {
+    //var query = generate({
+      //'text': 'test',
+      //'categories': ['retail','food']
+    //});
+
+    //var compiled = JSON.parse( JSON.stringify( query ) );
+    //var expected = require('../fixture/search_with_category_filtering');
+
+    //t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    //t.end();
+  //});
 };
 
 module.exports.all = function (tape, common) {

From 01d033848661aaab71a9d5dec9aaf1c98ad3ff71 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 3 Aug 2016 16:13:33 -0400
Subject: [PATCH 21/82] added text_parser that uses addressit format for
 autocomplete

rather than wholesale converting to libpostal in one release, the decision was made to only use libpostal for /search and not /autocomplete.  Until such time that libpostal can be used for parsing autocomplete queries, text_parser_autocomplete.js will contain the converter between addressit and internal parsing format.
---
 query/autocomplete.js             |  2 +-
 query/text_parser_autocomplete.js | 99 +++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 query/text_parser_autocomplete.js

diff --git a/query/autocomplete.js b/query/autocomplete.js
index 33f394f5..24e4d6ac 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -1,7 +1,7 @@
 
 var peliasQuery = require('pelias-query'),
     defaults = require('./autocomplete_defaults'),
-    textParser = require('./text_parser'),
+    textParser = require('./text_parser_autocomplete'),
     check = require('check-types');
 
 // additional views (these may be merged in to pelias/query at a later date)
diff --git a/query/text_parser_autocomplete.js b/query/text_parser_autocomplete.js
new file mode 100644
index 00000000..00e60724
--- /dev/null
+++ b/query/text_parser_autocomplete.js
@@ -0,0 +1,99 @@
+
+var logger = require('pelias-logger').get('api');
+var placeTypes = require('../helper/placeTypes');
+
+/*
+This list should only contain admin fields we are comfortable matching in the case
+when we can't identify parts of an address. This shouldn't contain fields like country_a
+or postalcode because we should only try to match those when we're sure that's what they are.
+ */
+var adminFields = placeTypes.concat([
+  'region_a'
+]);
+
+/**
+  @todo: refactor me
+**/
+
+// all the address parsing logic
+function addParsedVariablesToQueryVariables( parsed_text, vs ){
+
+  // is it a street address?
+  var isStreetAddress = parsed_text.hasOwnProperty('number') && parsed_text.hasOwnProperty('street');
+  if( isStreetAddress ){
+    vs.var( 'input:name', parsed_text.number + ' ' + parsed_text.street );
+  }
+
+  // ?
+  else if( parsed_text.admin_parts ) {
+    vs.var( 'input:name', parsed_text.name );
+  }
+
+  // ?
+  else {
+    logger.warn( 'chaos monkey asks: what happens now?' );
+    logger.warn( parsed_text );
+    try{ throw new Error(); } catch(e){ logger.warn( e.stack ); } // print a stack trace
+  }
+
+  // ==== add parsed matches [address components] ====
+
+  // house number
+  if( parsed_text.hasOwnProperty('number') ){
+    vs.var( 'input:housenumber', parsed_text.number );
+  }
+
+  // street name
+  if( parsed_text.hasOwnProperty('street') ){
+    vs.var( 'input:street', parsed_text.street );
+  }
+
+  // postal code
+  if( parsed_text.hasOwnProperty('postalcode') ){
+    vs.var( 'input:postcode', parsed_text.postalcode );
+  }
+
+  // ==== add parsed matches [admin components] ====
+
+  // city
+  if( parsed_text.hasOwnProperty('city') ){
+    vs.var( 'input:county', parsed_text.city );
+  }
+
+  // state
+  if( parsed_text.hasOwnProperty('state') ){
+    vs.var( 'input:region_a', parsed_text.state );
+  }
+
+  // country
+  if( parsed_text.hasOwnProperty('country') ){
+    vs.var( 'input:country_a', parsed_text.country );
+  }
+
+  // ==== deal with the 'leftover' components ====
+  // @todo: clean up this code
+
+  // a concept called 'leftovers' which is just 'admin_parts' /or 'regions'.
+  var leftoversString = '';
+  if( parsed_text.hasOwnProperty('admin_parts') ){
+    leftoversString = parsed_text.admin_parts;
+  }
+  else if( parsed_text.hasOwnProperty('regions') ){
+    leftoversString = parsed_text.regions.join(' ');
+  }
+
+  // if we have 'leftovers' then assign them to any fields which
+  // currently don't have a value assigned.
+  if( leftoversString.length ){
+
+    // cycle through fields and set fields which
+    // are still currently unset
+    adminFields.forEach( function( key ){
+      if( !vs.isset( 'input:' + key ) ){
+        vs.var( 'input:' + key, leftoversString );
+      }
+    });
+  }
+}
+
+module.exports = addParsedVariablesToQueryVariables;

From b612b2750ee4511c9cb2c91e6ec5ec7a7e3a2cb3 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 3 Aug 2016 17:39:49 -0400
Subject: [PATCH 22/82] added addressit support for autocomplete via separatee
 text sanitiser

this commit combines the other sanitiser/_text.js and addressit parser logic from text-analyzer into one module for easier integration until such time that libpostal is ready for autocomplete
---
 package.json                              |   1 +
 sanitiser/_text_autocomplete.js           | 107 +++++++++
 test/unit/sanitiser/_text_autocomplete.js | 276 ++++++++++++++++++++++
 3 files changed, 384 insertions(+)
 create mode 100644 sanitiser/_text_autocomplete.js
 create mode 100644 test/unit/sanitiser/_text_autocomplete.js

diff --git a/package.json b/package.json
index dc099a12..2c4b940f 100644
--- a/package.json
+++ b/package.json
@@ -35,6 +35,7 @@
     "node": ">=0.10.26"
   },
   "dependencies": {
+    "addressit": "git://github.com/dianashk/addressit.git#temp",
     "async": "^2.0.0",
     "check-types": "^7.0.0",
     "elasticsearch": "^11.0.0",
diff --git a/sanitiser/_text_autocomplete.js b/sanitiser/_text_autocomplete.js
new file mode 100644
index 00000000..5578c4b1
--- /dev/null
+++ b/sanitiser/_text_autocomplete.js
@@ -0,0 +1,107 @@
+var check = require('check-types');
+var parser = require('addressit');
+var extend = require('extend');
+var _      = require('lodash');
+var logger = require('pelias-logger').get('api');
+
+// validate texts, convert types and apply defaults
+function sanitize( raw, clean ){
+
+  // error & warning messages
+  var messages = { errors: [], warnings: [] };
+
+  // invalid input 'text'
+  if( !check.nonEmptyString( raw.text ) ){
+    messages.errors.push('invalid param \'text\': text length, must be >0');
+  }
+
+  // valid input 'text'
+  else {
+
+    // valid text
+    clean.text = raw.text;
+
+    // parse text with query parser
+    var parsed_text = parse(clean.text);
+    if (check.assigned(parsed_text)) {
+      clean.parsed_text = parsed_text;
+    }
+  }
+
+  return messages;
+}
+
+// export function
+module.exports = sanitize;
+
+
+
+// this is the addressit functionality from https://github.com/pelias/text-analyzer/blob/master/src/addressItParser.js
+var DELIM = ',';
+
+function parse(query) {
+  var getAdminPartsBySplittingOnDelim = function(queryParts) {
+    // naive approach - for admin matching during query time
+    // split 'flatiron, new york, ny' into 'flatiron' and 'new york, ny'
+
+    var address = {};
+
+    if (queryParts.length > 1) {
+      address.name = queryParts[0].trim();
+
+      // 1. slice away all parts after the first one
+      // 2. trim spaces from each part just in case
+      // 3. join the parts back together with appropriate delimiter and spacing
+      address.admin_parts = queryParts.slice(1)
+                                .map(function (part) { return part.trim(); })
+                                .join(DELIM + ' ');
+    }
+
+    return address;
+  };
+
+  var getAddressParts = function(query) {
+    // perform full address parsing
+    // except on queries so short they obviously can't contain an address
+    if (query.length > 3) {
+      return parser( query );
+    }
+  };
+
+  var queryParts = query.split(DELIM);
+
+  var addressWithAdminParts  = getAdminPartsBySplittingOnDelim(queryParts);
+  var addressWithAddressParts= getAddressParts(queryParts.join(DELIM + ' '));
+
+  var parsedAddress  = extend(addressWithAdminParts,
+                              addressWithAddressParts);
+
+  var address_parts  =  [ 'name',
+                          'number',
+                          'street',
+                          'city',
+                          'state',
+                          'country',
+                          'postalcode',
+                          'regions',
+                          'admin_parts'
+                        ];
+
+  var parsed_text = {};
+
+  address_parts.forEach(function(part){
+    if (parsedAddress[part]) {
+      parsed_text[part] = parsedAddress[part];
+    }
+  });
+
+  // if all we found was regions, ignore it as it is not enough information to make smarter decisions
+  if (Object.keys(parsed_text).length === 1 && !_.isUndefined(parsed_text.regions))
+  {
+    logger.info('Ignoring address parser output, regions only');
+    return null;
+  }
+
+  return parsed_text;
+
+}
diff --git a/test/unit/sanitiser/_text_autocomplete.js b/test/unit/sanitiser/_text_autocomplete.js
new file mode 100644
index 00000000..a8911b69
--- /dev/null
+++ b/test/unit/sanitiser/_text_autocomplete.js
@@ -0,0 +1,276 @@
+var sanitiser = require('../../../sanitiser/_text_autocomplete');
+var type_mapping = require('../../../helper/type_mapping');
+
+module.exports.tests = {};
+
+module.exports.tests.text_parser = function(test, common) {
+  test('short input text has admin layers set ', function(t) {
+    var raw = {
+      text: 'emp'  //start of empire state building
+    };
+    var clean = {
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  var queries = [
+    { name: 'soho', admin_parts: 'new york' },
+    { name: 'chelsea', admin_parts: 'london' },
+    { name: '123 main', admin_parts: 'new york' }
+  ];
+
+  queries.forEach(function (query) {
+    test('naive parsing ' + query, function(t) {
+      var raw = {
+        text: query.name + ', ' + query.admin_parts
+      };
+      var clean = {};
+
+      var expected_clean = {
+        text: query.name + ', ' + query.admin_parts,
+        parsed_text: {
+          name: query.name,
+          regions: [ query.name, query.admin_parts ],
+          admin_parts: query.admin_parts
+        }
+      };
+
+      var messages = sanitiser(raw, clean);
+
+      t.deepEqual(messages, { errors: [], warnings: [] } );
+      t.deepEqual(clean, expected_clean);
+      t.end();
+
+    });
+
+    test('naive parsing ' + query + ' without spaces', function(t) {
+      var raw = {
+        text: query.name + ',' + query.admin_parts
+      };
+      var clean = {};
+
+      var expected_clean = {
+        text: query.name + ',' + query.admin_parts,
+        parsed_text: {
+          name: query.name,
+          regions: [ query.name, query.admin_parts ],
+          admin_parts: query.admin_parts
+        }
+      };
+
+      var messages = sanitiser(raw, clean);
+
+      t.deepEqual(messages, { errors: [], warnings: [] } );
+      t.deepEqual(clean, expected_clean);
+      t.end();
+
+    });
+
+  });
+
+  test('query with one token', function (t) {
+    var raw = {
+      text: 'yugolsavia'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: 'yugolsavia'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('query with two tokens, no numbers', function (t) {
+    var raw = {
+      text: 'small town'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: 'small town'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('query with two tokens, number first', function (t) {
+    var raw = {
+      text: '123 main'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: '123 main'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('query with two tokens, number second', function (t) {
+    var raw = {
+      text: 'main 123'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: 'main 123'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('query with many tokens', function(t) {
+    var raw = {
+      text: 'main particle new york'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: 'main particle new york'
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('valid address, house number', function(t) {
+    var raw = {
+      text: '123 main st new york ny'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: '123 main st new york ny',
+      parsed_text: {
+        number: '123',
+        street: 'main st',
+        state: 'NY',
+        regions: [ 'new york' ]
+      }
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+  test('valid address, zipcode', function(t) {
+    var raw = {
+      text: '123 main st new york ny 10010'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: '123 main st new york ny 10010',
+      parsed_text: {
+        number: '123',
+        street: 'main st',
+        state: 'NY',
+        postalcode: '10010',
+        regions: [ 'new york' ]
+      }
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+  });
+
+  test('valid address with leading 0s in zipcode', function(t) {
+    var raw = {
+      text: '339 W Main St, Cheshire, 06410'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: '339 W Main St, Cheshire, 06410',
+      parsed_text: {
+        name: '339 W Main St',
+        number: '339',
+        street: 'W Main St',
+        postalcode: '06410',
+        regions: [ 'Cheshire' ],
+        admin_parts: 'Cheshire, 06410'
+      }
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+  });
+
+  test('valid address without spaces after commas', function(t) {
+    var raw = {
+      text: '339 W Main St,Lancaster,PA'
+    };
+    var clean = {};
+
+    var expected_clean = {
+      text: '339 W Main St,Lancaster,PA',
+      parsed_text: {
+        name: '339 W Main St',
+        number: '339',
+        street: 'W Main St',
+        state: 'PA',
+        regions: [ 'Lancaster' ],
+        admin_parts: 'Lancaster, PA'
+      }
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEqual(messages, { errors: [], warnings: [] } );
+    t.deepEqual(clean, expected_clean);
+    t.end();
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANITISER _text: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From 486f25663af3b5e25b9d64c43bb1292b801f7841 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 3 Aug 2016 17:43:06 -0400
Subject: [PATCH 23/82] switched autocomplete sanitiser wrapper to addressit
 text sanitiser

---
 sanitiser/autocomplete.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanitiser/autocomplete.js b/sanitiser/autocomplete.js
index a7ee68f6..025485cf 100644
--- a/sanitiser/autocomplete.js
+++ b/sanitiser/autocomplete.js
@@ -3,7 +3,7 @@ var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
-      text: require('../sanitiser/_text'),
+      text: require('../sanitiser/_text_autocomplete'),
       tokenizer: require('../sanitiser/_tokenizer'),
       size: require('../sanitiser/_size')(10, 10, 10),
       layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),

From 85dab16869fdffa6182a2cd909131465af52b4bf Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 4 Aug 2016 16:55:55 -0400
Subject: [PATCH 24/82] refactored search sanitiser tests

the tests removed were testing far more than was appropriate since the functionality of the individual sanitisers is tested elsewhere.  in this case, the test has been reduced to just testing that all the sanitisers were actually called.
---
 query/search.js               |   5 +-
 sanitiser/search.js           |   6 +-
 test/unit/sanitiser/search.js | 383 +++++++++-------------------------
 3 files changed, 99 insertions(+), 295 deletions(-)

diff --git a/query/search.js b/query/search.js
index c7a26758..29c67876 100644
--- a/query/search.js
+++ b/query/search.js
@@ -123,8 +123,9 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  var q = query.render(vs);
-  console.log(JSON.stringify(q, null, 2));
+  var q = getQuery(vs);
+
+  //console.log(JSON.stringify(q, null, 2));
 
   return q;
 }
diff --git a/sanitiser/search.js b/sanitiser/search.js
index 7fcc6ab6..feceb2ef 100644
--- a/sanitiser/search.js
+++ b/sanitiser/search.js
@@ -8,7 +8,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       size: require('../sanitiser/_size')(/* use defaults*/),
       layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
-      // depends on the layers and sources sanitisers, must be run after them
+      // // depends on the layers and sources sanitisers, must be run after them
       sources_and_layers: require('../sanitiser/_sources_and_layers'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),
@@ -18,10 +18,6 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
-// export sanitize for testing
-module.exports.sanitize = sanitize;
-module.exports.sanitiser_list = sanitizers;
-
 // middleware
 module.exports.middleware = function( req, res, next ){
   sanitize( req, function( err, clean ){
diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js
index 35dbcda5..3c5c684d 100644
--- a/test/unit/sanitiser/search.js
+++ b/test/unit/sanitiser/search.js
@@ -1,299 +1,106 @@
-var extend = require('extend'),
-    search  = require('../../../sanitiser/search'),
-    text_analyzer = require('pelias-text-analyzer'),
-    sanitize = search.sanitize,
-    middleware = search.middleware,
-    defaultError = 'invalid param \'text\': text length, must be >0';
-// these are the default values you would expect when no input params are specified.
-var emptyClean = { private: false, size: 10 };
+var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
 
-module.exports.tests.interface = function(test, common) {
-  test('sanitize interface', function(t) {
-    t.equal(typeof sanitize, 'function', 'sanitize is a function');
-    t.equal(sanitize.length, 2, 'sanitize interface');
-    t.end();
-  });
-  test('middleware interface', function(t) {
-    t.equal(typeof middleware, 'function', 'middleware is a function');
-    t.equal(middleware.length, 3, 'sanitize has a valid middleware');
-    t.end();
-  });
-};
-
-module.exports.tests.sanitisers = function(test, common) {
-  test('check sanitiser list', function (t) {
-    var expected = ['quattroshapes_deprecation', 'singleScalarParameters', 'text', 'size',
-      'layers', 'sources', 'sources_and_layers', 'private', 'geo_search', 'boundary_country', 'categories' ];
-    t.deepEqual(Object.keys(search.sanitiser_list), expected);
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_invalid_text = function(test, common) {
-  test('invalid text', function(t) {
-    var invalid = [ '', 100, null, undefined ];
-    invalid.forEach( function( text ){
-      var req = { query: { text: text } };
-      sanitize(req, function(){
-        t.equal(req.errors[0], 'invalid param \'text\': text length, must be >0', text + ' is an invalid text');
-        t.deepEqual(req.clean, emptyClean, 'clean only has default values set');
-      });
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitise_valid_text = function(test, common) {
-  test('valid short text', function(t) {
-    var req = { query: { text: 'a' } };
-    sanitize(req, function(){
-      t.equal(req.errors[0], undefined, 'no error');
-    });
-    t.end();
-  });
-
-  test('valid not-quite-as-short text', function(t) {
-    var req = { query: { text: 'aa' } };
-    sanitize(req, function(){
-      t.equal(req.errors[0], undefined, 'no error');
-    });
-    t.end();
-  });
-
-  test('valid longer text', function(t) {
-    var req = { query: { text: 'aaaaaaaa' } };
-    sanitize(req, function(){
-      t.equal(req.errors[0], undefined, 'no error');
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_text_with_delim = function(test, common) {
-  var texts = [ 'a,bcd', '123 main st, region', ',,,', ' ' ];
-
-  test('valid texts with a comma', function(t) {
-    texts.forEach( function( text ){
-      var req = { query: { text: text } };
-      sanitize( req, function( ){
-        var expected_text = text;
-
-        var expected_parsed_text = text_analyzer.parse(text);
-        t.equal(req.errors[0], undefined, 'no error');
-        t.equal(req.clean.parsed_text.name, expected_parsed_text.name, 'clean name set correctly');
-        t.equal(req.clean.text, expected_text, 'text should match');
-
-      });
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_private_no_value = function(test, common) {
-  test('default private should be set to true', function(t) {
-    var req = { query: { text: 'test' } };
-    sanitize(req, function(){
-      t.equal(req.clean.private, false, 'private set to false');
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_private_explicit_true_value = function(test, common) {
-  test('explicit private should be set to true', function(t) {
-    var req = { query: { text: 'test', private: true } };
-    sanitize(req, function(){
-      t.equal(req.clean.private, true, 'private set to true');
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_private_explicit_false_value = function(test, common) {
-  test('explicit private should be set to false', function(t) {
-    var req = { query: { text: 'test', private: false } };
-    sanitize(req, function(){
-      t.equal(req.clean.private, false, 'private set to false');
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_lat = function(test, common) {
-  var valid_lats = [ 0, 45, 90, -0, '0', '45', '90', -181, -120, -91, 91, 120, 181  ];
-  test('valid lat', function(t) {
-    valid_lats.forEach( function( lat ){
-      var req = { query: { text: 'test', 'focus.point.lat': lat, 'focus.point.lon': 0 } };
-      sanitize(req, function(){
-        var expected_lat = parseFloat( lat );
-        t.equal(req.errors[0], undefined, 'no error');
-      });
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_lon = function(test, common) {
-  var lons = {
-    valid: [ -381, -181, -180, -1, -0, 0, 45, 90, '-180', '0', '180', 181 ]
-  };
-  test('valid lon', function(t) {
-    lons.valid.forEach( function( lon ){
-      var req = { query: { text: 'test', 'focus.point.lat': 0, 'focus.point.lon': lon } };
-      sanitize( req, function(){
-        var expected_lon = parseFloat( lon );
-        t.equal(req.errors[0], undefined, 'no error');
-      });
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_optional_geo = function(test, common) {
-  test('no lat/lon', function(t) {
-    var req = { query: { text: 'test' } };
-    sanitize(req, function(){
-      t.equal(req.errors[0], undefined, 'no error');
-      t.equal(req.clean['focus.point.lat'], undefined, 'clean set without lat');
-      t.equal(req.clean['focus.point.lon'], undefined, 'clean set without lon');
-    });
-    t.end();
-  });
-  test('no lat', function(t) {
-    var req = { query: { text: 'test', 'focus.point.lon': 0 } };
-    sanitize(req, function(){
-      var expected_lon = 0;
-      t.equal(req.errors[0], 'parameters focus.point.lat and focus.point.lon must both be specified');
-      t.equal(req.clean['focus.point.lat'], undefined);
-      t.equal(req.clean['focus.point.lon'], undefined);
-    });
-    t.end();
-  });
-  test('no lon', function(t) {
-    var req = { query: { text: 'test', 'focus.point.lat': 0 } };
-    sanitize(req, function(){
-      var expected_lat = 0;
-      t.equal(req.errors[0], 'parameters focus.point.lat and focus.point.lon must both be specified');
-      t.equal(req.clean['focus.point.lat'], undefined);
-      t.equal(req.clean['focus.point.lon'], undefined);
-    });
-    t.end();
-  });
-};
-
-module.exports.tests.sanitize_bounding_rect = function(test, common) {
-  test('valid bounding rect', function(t) {
-    var req = {
-      query: {
-        text: 'test',
-        'boundary.rect.min_lat': -40.659,
-        'boundary.rect.max_lat': -41.614,
-        'boundary.rect.min_lon': 174.612,
-        'boundary.rect.max_lon': 176.333
-      }
-    };
-
-    sanitize(req, function(){
-      t.equal(req.errors[0], undefined, 'no error');
-      t.equal(req.clean['boundary.rect.min_lon'], parseFloat(req.query['boundary.rect.min_lon']));
-      t.equal(req.clean['boundary.rect.max_lat'], parseFloat(req.query['boundary.rect.max_lat']));
-      t.equal(req.clean['boundary.rect.max_lon'], parseFloat(req.query['boundary.rect.max_lon']));
-      t.equal(req.clean['boundary.rect.min_lat'], parseFloat(req.query['boundary.rect.min_lat']));
+module.exports.tests.sanitize = function(test, common) {
+  test('verify that all sanitisers were called as expected', function(t) {
+    var called_sanitisers = [];
+
+    // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
+    //  were all called correctly
+    var search = proxyquire('../../../sanitiser/search', {
+      '../sanitiser/_deprecate_quattroshapes': function() {
+        called_sanitisers.push('_deprecate_quattroshapes');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_single_scalar_parameters': function() {
+        called_sanitisers.push('_single_scalar_parameters');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_text': function() {
+        called_sanitisers.push('_text');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_size': function() {
+        if (arguments.length === 0) {
+          return function() {
+            called_sanitisers.push('_size');
+            return { errors: [], warnings: [] };
+          };
+
+        } else {
+          throw new Error('should not have passed any parameters to _size');
+        }
+
+      },
+      '../sanitiser/_targets': function(type) {
+        if (['layers', 'sources'].indexOf(type) !== -1) {
+          return function() {
+            called_sanitisers.push('_targets/' + type);
+            return { errors: [], warnings: [] };
+          };
+
+        }
+        else {
+          throw new Error('incorrect parameters passed to _targets');
+        }
+
+      },
+      '../sanitiser/_sources_and_layers': function() {
+        called_sanitisers.push('_sources_and_layers');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_flag_bool': function() {
+        if (arguments[0] === 'private' && arguments[1] === false) {
+          return function() {
+            called_sanitisers.push('_flag_bool');
+            return { errors: [], warnings: [] };
+          };
+
+        }
+        else {
+          throw new Error('incorrect parameters passed to _flag_bool');
+        }
+
+      },
+      '../sanitiser/_geo_search': function() {
+        called_sanitisers.push('_geo_search');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_boundary_country': function() {
+        called_sanitisers.push('_boundary_country');
+        return { errors: [], warnings: [] };
+      },
+      '../sanitiser/_categories': function() {
+        called_sanitisers.push('_categories');
+        return { errors: [], warnings: [] };
+      },
+    });
+
+    var expected_sanitisers = [
+      '_deprecate_quattroshapes',
+      '_single_scalar_parameters',
+      '_text',
+      '_size',
+      '_targets/layers',
+      '_targets/sources',
+      '_sources_and_layers',
+      '_flag_bool',
+      '_geo_search',
+      '_boundary_country',
+      '_categories'
+    ];
+
+    var req = {};
+    var res = {};
+
+    search.middleware(req, res, function(){
+      t.deepEquals(called_sanitisers, expected_sanitisers);
       t.end();
     });
   });
 };
 
-module.exports.tests.sanitize_size = function(test, common) {
-  test('invalid size value', function(t) {
-    var req = { query: { size: 'a', text: 'test', lat: 0, lon: 0 } };
-    sanitize(req, function(){
-      t.equal(req.clean.size, 10, 'default size set');
-      t.end();
-    });
-  });
-  test('below min size value', function(t) {
-    var req = { query: { size: -100, text: 'test', lat: 0, lon: 0 } };
-    sanitize(req, function(){
-      t.equal(req.clean.size, 1, 'min size set');
-      t.end();
-    });
-  });
-  test('above max size value', function(t) {
-    var req = { query: { size: 9999, text: 'test', lat: 0, lon: 0 } };
-    sanitize(req, function(){
-      t.equal(req.clean.size, 40, 'max size set');
-      t.end();
-    });
-  });
-};
-
-module.exports.tests.sanitize_private = function(test, common) {
-  var invalid_values = [null, -1, 123, NaN, 'abc'];
-  invalid_values.forEach(function(value) {
-    test('invalid private param ' + value, function(t) {
-      var req = { query: { text: 'test', lat: 0, lon: 0, 'private': value } };
-      sanitize(req, function(){
-        t.equal(req.clean.private, false, 'default private set (to false)');
-        t.end();
-      });
-    });
-  });
-
-  var valid_values = ['true', true, 1, '1'];
-  valid_values.forEach(function(value) {
-    test('valid private ' + value, function(t) {
-      var req = { query: { text: 'test', 'private': value } };
-      sanitize(req, function(){
-        t.equal(req.clean.private, true, 'private set to true');
-        t.end();
-      });
-    });
-  });
-
-  var valid_false_values = ['false', false, 0, '0'];
-  valid_false_values.forEach(function(value) {
-    test('test setting false explicitly ' + value, function(t) {
-      var req = { query: { text: 'test', 'private': value } };
-      sanitize(req, function(){
-        t.equal(req.clean.private, false, 'private set to false');
-        t.end();
-      });
-    });
-  });
-
-  test('test default behavior', function(t) {
-    var req = { query: { text: 'test' } };
-    sanitize(req, function(){
-      t.equal(req.clean.private, false, 'private set to false');
-      t.end();
-    });
-  });
-};
-
-module.exports.tests.invalid_params = function(test, common) {
-  test('invalid text params', function(t) {
-    var req = { query: {} };
-    sanitize( req, function(){
-      t.equal(req.errors[0], defaultError, 'handle invalid params gracefully');
-      t.end();
-    });
-  });
-};
-
-module.exports.tests.middleware_success = function(test, common) {
-  test('middleware success', function(t) {
-    var req = { query: { text: 'test' }};
-    var next = function( message ){
-      t.deepEqual(req.errors, [], 'no error messages set');
-      t.end();
-    };
-    middleware( req, undefined, next );
-  });
-};
-
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {

From af7de7613f4d7595a4d89ba00d14457104a175ae Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 5 Aug 2016 09:13:57 -0400
Subject: [PATCH 25/82] updated to reflect @missinglink's comments

the rebase was brutal, i thought that i caught everything, thx for checking
---
 index.js                        | 1 +
 middleware/trimByGranularity.js | 2 +-
 sanitiser/search.js             | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index d9503bf0..42116f53 100644
--- a/index.js
+++ b/index.js
@@ -2,6 +2,7 @@
 var cluster = require('cluster'),
     app = require('./app'),
     port = ( process.env.PORT || 3100 ),
+    // when pelias/api#601 is done this can be changed to `true`
     multicore = false;
 
 /** cluster webserver across all cores **/
diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js
index 9594e879..1fade0cc 100644
--- a/middleware/trimByGranularity.js
+++ b/middleware/trimByGranularity.js
@@ -10,7 +10,7 @@ var _ = require('lodash');
 // - state
 //
 // Because the address matched, we're not interested in city+state or state, so
-// this component removes that aren't the most granular.  
+// this component removes results that aren't the most granular.
 
 // layers in increasing order of granularity
 var layers = [
diff --git a/sanitiser/search.js b/sanitiser/search.js
index feceb2ef..130de40f 100644
--- a/sanitiser/search.js
+++ b/sanitiser/search.js
@@ -8,7 +8,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       size: require('../sanitiser/_size')(/* use defaults*/),
       layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
-      // // depends on the layers and sources sanitisers, must be run after them
+      // depends on the layers and sources sanitisers, must be run after them
       sources_and_layers: require('../sanitiser/_sources_and_layers'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_search: require('../sanitiser/_geo_search'),

From 0a59e2750ffd868153efac4115d7e7a9387986c7 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 12 Aug 2016 16:04:27 -0400
Subject: [PATCH 26/82] inject `query` to use

---
 routes/v1.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routes/v1.js b/routes/v1.js
index dd5ba0a9..b066cca4 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -63,7 +63,7 @@ function addRoutes(app, peliasConfig) {
     search: createRouter([
       sanitisers.search.middleware,
       middleware.calcSize(),
-      controllers.search(peliasConfig),
+      controllers.search(peliasConfig, undefined, require('../query/search')),
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),

From 45a0755f7d31175c913398763ee47c140778af6b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 15 Aug 2016 09:53:13 -0400
Subject: [PATCH 27/82] added 2nd /search query call for fallback to existing
 behavior

while we're refining our process for integrating libpostal, the ES querying behavior will be:

- call ES with the fallback/geodisambiguation query
- if there are 0 results, call ES with the existing behavior
---
 routes/v1.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/routes/v1.js b/routes/v1.js
index b066cca4..6dc0cdbf 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -63,7 +63,11 @@ function addRoutes(app, peliasConfig) {
     search: createRouter([
       sanitisers.search.middleware,
       middleware.calcSize(),
+      // 2nd parameter is `backend` which gets initialized internally
+      // 3rd parameter is which query module to use, use fallback/geodisambiguation
+      //  first, then use  if first query didn't return anything 
       controllers.search(peliasConfig, undefined, require('../query/search')),
+      controllers.search(peliasConfig, undefined, require('../query/search_original')),
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),

From e01ef7ac4d6ad2552f9df60ba5d2faf51e5c2273 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 15 Aug 2016 10:49:57 -0400
Subject: [PATCH 28/82] copied the existing search query logic back in

---
 query/search_original.js | 131 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)
 create mode 100644 query/search_original.js

diff --git a/query/search_original.js b/query/search_original.js
new file mode 100644
index 00000000..5ab96248
--- /dev/null
+++ b/query/search_original.js
@@ -0,0 +1,131 @@
+var peliasQuery = require('pelias-query'),
+    defaults = require('./search_defaults'),
+    textParser = require('./text_parser'),
+    check = require('check-types');
+
+var placeTypes = require('../helper/placeTypes');
+
+// region_a is also an admin field. addressit tries to detect
+// region_a, in which case we use a match query specifically for it.
+// but address it doesn't know about all of them so it helps to search
+// against this with the other admin parts as a fallback
+var adminFields = placeTypes.concat(['region_a']);
+
+//------------------------------
+// general-purpose search query
+//------------------------------
+var query = new peliasQuery.layout.FilteredBooleanQuery();
+
+// mandatory matches
+query.score( peliasQuery.view.boundary_country, 'must' );
+query.score( peliasQuery.view.ngrams, 'must' );
+
+// scoring boost
+query.score( peliasQuery.view.phrase );
+query.score( peliasQuery.view.focus( peliasQuery.view.phrase ) );
+query.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
+query.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
+
+// address components
+query.score( peliasQuery.view.address('housenumber') );
+query.score( peliasQuery.view.address('street') );
+query.score( peliasQuery.view.address('postcode') );
+
+// admin components
+// country_a and region_a are left as matches here because the text-analyzer
+// can sometimes detect them, in which case a query more specific than a
+// multi_match is appropriate.
+query.score( peliasQuery.view.admin('country_a') );
+query.score( peliasQuery.view.admin('region_a') );
+query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') );
+
+// non-scoring hard filters
+query.filter( peliasQuery.view.boundary_circle );
+query.filter( peliasQuery.view.boundary_rect );
+query.filter( peliasQuery.view.sources );
+query.filter( peliasQuery.view.layers );
+query.filter( peliasQuery.view.categories );
+
+// --------------------------------
+
+/**
+  map request variables to query variables for all inputs
+  provided by this HTTP request.
+**/
+function generateQuery( clean ){
+
+  var vs = new peliasQuery.Vars( defaults );
+
+  // input text
+  vs.var( 'input:name', clean.text );
+
+  // sources
+  vs.var( 'sources', clean.sources);
+
+  // layers
+  vs.var( 'layers', clean.layers);
+
+  // categories
+  if (clean.categories) {
+    vs.var('input:categories', clean.categories);
+  }
+
+  // size
+  if( clean.querySize ) {
+    vs.var( 'size', clean.querySize );
+  }
+
+  // focus point
+  if( check.number(clean['focus.point.lat']) &&
+      check.number(clean['focus.point.lon']) ){
+    vs.set({
+      'focus:point:lat': clean['focus.point.lat'],
+      'focus:point:lon': clean['focus.point.lon']
+    });
+  }
+
+  // boundary rect
+  if( check.number(clean['boundary.rect.min_lat']) &&
+      check.number(clean['boundary.rect.max_lat']) &&
+      check.number(clean['boundary.rect.min_lon']) &&
+      check.number(clean['boundary.rect.max_lon']) ){
+    vs.set({
+      'boundary:rect:top': clean['boundary.rect.max_lat'],
+      'boundary:rect:right': clean['boundary.rect.max_lon'],
+      'boundary:rect:bottom': clean['boundary.rect.min_lat'],
+      'boundary:rect:left': clean['boundary.rect.min_lon']
+    });
+  }
+
+  // boundary circle
+  // @todo: change these to the correct request variable names
+  if( check.number(clean['boundary.circle.lat']) &&
+      check.number(clean['boundary.circle.lon']) ){
+    vs.set({
+      'boundary:circle:lat': clean['boundary.circle.lat'],
+      'boundary:circle:lon': clean['boundary.circle.lon']
+    });
+
+    if( check.number(clean['boundary.circle.radius']) ){
+      vs.set({
+        'boundary:circle:radius': Math.round( clean['boundary.circle.radius'] ) + 'km'
+      });
+    }
+  }
+
+  // boundary country
+  if( check.string(clean['boundary.country']) ){
+    vs.set({
+      'boundary:country': clean['boundary.country']
+    });
+  }
+
+  // run the address parser
+  if( clean.parsed_text ){
+    textParser( clean.parsed_text, vs );
+  }
+
+  return query.render( vs );
+}
+
+module.exports = generateQuery;

From 4952a0ed2bfa0f522c4f982865f88eec71a6c5b0 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 15 Aug 2016 11:00:34 -0400
Subject: [PATCH 29/82] added condition to exit early if there are already
 results in `res`

in order to accommodate falling back to the existing search strategy, the search controller must exit early if the `res.data` already exists.
---
 controller/search.js           |  7 +++++++
 test/unit/controller/search.js | 22 ++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/controller/search.js b/controller/search.js
index 271d2899..70d403a8 100644
--- a/controller/search.js
+++ b/controller/search.js
@@ -17,6 +17,13 @@ function setup( config, backend, query ){
       return next();
     }
 
+    // do not run controller if there are already results
+    // this was added during libpostal integration.  if the libpostal parse/query
+    // doesn't return anything then fallback to old search-engine-y behavior
+    if (res && res.hasOwnProperty('data')) {
+      return next();
+    }
+
     var cleanOutput = _.cloneDeep(req.clean);
     if (logging.isDNT(req)) {
       cleanOutput = logging.removeFields(cleanOutput);
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 3f74fbd7..76f412c8 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -171,6 +171,28 @@ module.exports.tests.timeout = function(test, common) {
   });
 };
 
+module.exports.tests.existing_results = function(test, common) {
+  test('res with existing data should not call backend', function(t) {
+    var backend = function() {
+      throw new Error('backend should not have been called');
+    };
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
+
+    var req = { };
+    // the existence of `data` means that there are already results so
+    // don't call the backend/query
+    var res = { data: [] };
+
+    var next = function() {
+      t.deepEqual(res, {data: []});
+      t.end();
+    };
+    controller(req, res, next);
+
+  });
+
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {

From 2a9c25f8c4ae2a63f32e7faa5b58fe4d0213887d Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 15 Aug 2016 11:31:49 -0400
Subject: [PATCH 30/82] added condition that `res.data` must be non-empty to
 return early

---
 controller/search.js           | 2 +-
 test/unit/controller/search.js | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/controller/search.js b/controller/search.js
index 70d403a8..b4124acf 100644
--- a/controller/search.js
+++ b/controller/search.js
@@ -20,7 +20,7 @@ function setup( config, backend, query ){
     // do not run controller if there are already results
     // this was added during libpostal integration.  if the libpostal parse/query
     // doesn't return anything then fallback to old search-engine-y behavior
-    if (res && res.hasOwnProperty('data')) {
+    if (res && res.hasOwnProperty('data') && res.data.length > 0) {
       return next();
     }
 
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 76f412c8..0c8e40ae 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -181,10 +181,10 @@ module.exports.tests.existing_results = function(test, common) {
     var req = { };
     // the existence of `data` means that there are already results so
     // don't call the backend/query
-    var res = { data: [] };
+    var res = { data: [{}] };
 
     var next = function() {
-      t.deepEqual(res, {data: []});
+      t.deepEqual(res, {data: [{}]});
       t.end();
     };
     controller(req, res, next);

From 3b981d97ba2c19fec803e9a93d7789d0bd879361 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Mon, 15 Aug 2016 14:13:28 -0400
Subject: [PATCH 31/82] copied in tests from master for `search_original`

commented out the tests that break because i don't know how to fix them which will probably require help from @missinglink.
---
 test/unit/query/search_original.js | 198 +++++++++++++++++++++++++++++
 test/unit/run.js                   |   1 +
 2 files changed, 199 insertions(+)
 create mode 100644 test/unit/query/search_original.js

diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js
new file mode 100644
index 00000000..df0d2677
--- /dev/null
+++ b/test/unit/query/search_original.js
@@ -0,0 +1,198 @@
+var generate = require('../../../query/search_original');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof generate, 'function', 'valid function');
+    t.end();
+  });
+};
+
+module.exports.tests.query = function(test, common) {
+  test('valid search + focus + bbox', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+      'boundary.rect.min_lat': 47.47,
+      'boundary.rect.max_lon': -61.84,
+      'boundary.rect.max_lat': 11.51,
+      'boundary.rect.min_lon': -103.16,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus_bbox');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
+    t.end();
+  });
+
+  test('valid search + bbox', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'boundary.rect.min_lat': 47.47,
+      'boundary.rect.max_lon': -61.84,
+      'boundary.rect.max_lat': 11.51,
+      'boundary.rect.min_lon': -103.16,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_bbox');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_bbox');
+    t.end();
+  });
+
+  test('valid lingustic-only search', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_only');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_only');
+    t.end();
+  });
+
+  test('search search + focus', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus');
+    t.end();
+  });
+
+  test('search search + focus on null island', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 0, 'focus.point.lon': 0,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus_null_island');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
+    t.end();
+  });
+
+  // test('valid query with a full valid address', function(t) {
+  //   var query = generate({ text: '123 main st new york ny 10010 US',
+  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+  //     querySize: 10,
+  //     parsed_text: {
+  //       number: '123',
+  //       street: 'main st',
+  //       state: 'NY',
+  //       country: 'USA',
+  //       postalcode: '10010'
+  //     }
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_full_address');
+  //
+  //   var fs = require('fs');
+  //   fs.writeFileSync('actual.json', JSON.stringify(compiled, null, 2));
+  //   fs.writeFileSync('expected.json', JSON.stringify(expected, null, 2));
+  //
+  //   t.deepEqual(compiled, expected, 'search_full_address');
+  //   t.end();
+  // });
+  //
+  // test('valid query with partial address', function(t) {
+  //   var query = generate({ text: 'soho grand, new york',
+  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+  //     querySize: 10,
+  //     parsed_text: { name: 'soho grand',
+  //       state: 'NY',
+  //       regions: [ 'soho grand' ],
+  //       admin_parts: 'new york'
+  //     }
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_partial_address');
+  //
+  //   t.deepEqual(compiled, expected, 'search_partial_address');
+  //   t.end();
+  // });
+  //
+  // test('valid query with regions in address', function(t) {
+  //   var query = generate({ text: '1 water st manhattan ny',
+  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+  //     querySize: 10,
+  //     parsed_text: { number: '1',
+  //       street: 'water st',
+  //       state: 'NY',
+  //       regions: [ 'manhattan' ]
+  //     },
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_regions_address');
+  //
+  //   t.deepEqual(compiled, expected, 'search_regions_address');
+  //   t.end();
+  // });
+
+  test('valid boundary.country search', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      layers: ['test'],
+      'boundary.country': 'ABC'
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_boundary_country');
+
+    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
+    t.end();
+  });
+
+  test('valid sources filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'sources': ['test_source']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_source_filtering');
+
+    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
+    t.end();
+  });
+
+  test('categories filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'categories': ['retail','food']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_category_filtering');
+
+    t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('search query ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/test/unit/run.js b/test/unit/run.js
index ae9790c6..13035037 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -38,6 +38,7 @@ var tests = [
   require('./query/reverse_defaults'),
   require('./query/reverse'),
   require('./query/search'),
+  require('./query/search_original'),
   require('./sanitiser/_boundary_country'),
   require('./sanitiser/_flag_bool'),
   require('./sanitiser/_geo_common'),

From d8fb9323c2082f6d9e47f1d15b8d001bbbb65a52 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 12:56:58 -0400
Subject: [PATCH 32/82] added sanitiser wrapper for fallback to call addressit
 explicitly

---
 sanitiser/search_fallback.js           | 13 ++++++++
 test/unit/sanitiser/search_fallback.js | 41 ++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 sanitiser/search_fallback.js
 create mode 100644 test/unit/sanitiser/search_fallback.js

diff --git a/sanitiser/search_fallback.js b/sanitiser/search_fallback.js
new file mode 100644
index 00000000..382b548b
--- /dev/null
+++ b/sanitiser/search_fallback.js
@@ -0,0 +1,13 @@
+var sanitizeAll = require('../sanitiser/sanitizeAll'),
+    sanitizers = {
+      text: require('../sanitiser/_text_autocomplete')
+    };
+
+var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+
+// middleware
+module.exports.middleware = function( req, res, next ){
+  sanitize( req, function( err, clean ){
+    next();
+  });
+};
diff --git a/test/unit/sanitiser/search_fallback.js b/test/unit/sanitiser/search_fallback.js
new file mode 100644
index 00000000..d2ce4c01
--- /dev/null
+++ b/test/unit/sanitiser/search_fallback.js
@@ -0,0 +1,41 @@
+var proxyquire =  require('proxyquire').noCallThru();
+
+module.exports.tests = {};
+
+module.exports.tests.sanitize = function(test, common) {
+  test('verify that all sanitisers were called as expected', function(t) {
+    var called_sanitisers = [];
+
+    // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
+    //  were all called correctly
+    var search = proxyquire('../../../sanitiser/search_fallback', {
+      '../sanitiser/_text_autocomplete': function() {
+        called_sanitisers.push('_text_autocomplete');
+        return { errors: [], warnings: [] };
+      }
+    });
+
+    var expected_sanitisers = [
+      '_text_autocomplete'
+    ];
+
+    var req = {};
+    var res = {};
+
+    search.middleware(req, res, function(){
+      t.deepEquals(called_sanitisers, expected_sanitisers);
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SANITIZE /search_fallback ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From 40ed4e64c412db995ba96b915c59ec1f314b547e Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 12:58:25 -0400
Subject: [PATCH 33/82] added new test

---
 test/unit/run.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/unit/run.js b/test/unit/run.js
index 13035037..bddb9ecf 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -60,6 +60,7 @@ var tests = [
   require('./sanitiser/place'),
   require('./sanitiser/reverse'),
   require('./sanitiser/search'),
+  require('./sanitiser/search_fallback'),
   require('./sanitiser/wrap'),
   require('./service/mget'),
   require('./service/search')

From 8af8ba19f6175ad598b24dcf4d06502c5edb1190 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 13:11:36 -0400
Subject: [PATCH 34/82] added call for fallback sanitiser middleware

---
 routes/v1.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/routes/v1.js b/routes/v1.js
index 6dc0cdbf..833240e9 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -7,6 +7,7 @@ var sanitisers = {
   autocomplete: require('../sanitiser/autocomplete'),
   place: require('../sanitiser/place'),
   search: require('../sanitiser/search'),
+  search_fallback: require('../sanitiser/search_fallback'),
   reverse: require('../sanitiser/reverse'),
   nearby: require('../sanitiser/nearby')
 };
@@ -65,8 +66,9 @@ function addRoutes(app, peliasConfig) {
       middleware.calcSize(),
       // 2nd parameter is `backend` which gets initialized internally
       // 3rd parameter is which query module to use, use fallback/geodisambiguation
-      //  first, then use  if first query didn't return anything 
+      //  first, then use original search strategy if first query didn't return anything
       controllers.search(peliasConfig, undefined, require('../query/search')),
+      sanitisers.search_fallback.middleware,
       controllers.search(peliasConfig, undefined, require('../query/search_original')),
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),

From ad5d2bf36a02d1dab445e438318be99b5f65a384 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 13:55:56 -0400
Subject: [PATCH 35/82] add early exit condition to stop unneeded re-parsing

This is required in the case where the libpostal parse + ES search was successful (res.data is non-empty) so we want to skip falling back to the current production behavior.
---
 sanitiser/search_fallback.js           |  8 +++
 test/unit/sanitiser/search_fallback.js | 82 +++++++++++++++++++++++++-
 2 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/sanitiser/search_fallback.js b/sanitiser/search_fallback.js
index 382b548b..ebe6b53e 100644
--- a/sanitiser/search_fallback.js
+++ b/sanitiser/search_fallback.js
@@ -7,7 +7,15 @@ var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 
 // middleware
 module.exports.middleware = function( req, res, next ){
+  // if res.data already has results then don't call the _text_autocomplete sanitiser
+  // this has been put into place for when the libpostal integration way of querying
+  // ES doesn't return anything and we want to fallback to the old logic
+  if (res && res.hasOwnProperty('data') && res.data.length > 0) {
+    return next();
+  }
+
   sanitize( req, function( err, clean ){
     next();
   });
+
 };
diff --git a/test/unit/sanitiser/search_fallback.js b/test/unit/sanitiser/search_fallback.js
index d2ce4c01..2003b19f 100644
--- a/test/unit/sanitiser/search_fallback.js
+++ b/test/unit/sanitiser/search_fallback.js
@@ -3,7 +3,32 @@ var proxyquire =  require('proxyquire').noCallThru();
 module.exports.tests = {};
 
 module.exports.tests.sanitize = function(test, common) {
-  test('verify that all sanitisers were called as expected', function(t) {
+  test('verify that all sanitisers were called as expected when `res` is undefined', function(t) {
+    var called_sanitisers = [];
+
+    // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
+    //  were all called correctly
+    var search = proxyquire('../../../sanitiser/search_fallback', {
+      '../sanitiser/_text_autocomplete': function() {
+        called_sanitisers.push('_text_autocomplete');
+        return { errors: [], warnings: [] };
+      }
+    });
+
+    var expected_sanitisers = [
+      '_text_autocomplete'
+    ];
+
+    var req = {};
+
+    search.middleware(req, undefined, function(){
+      t.deepEquals(called_sanitisers, expected_sanitisers);
+      t.end();
+    });
+
+  });
+
+  test('verify that all sanitisers were called as expected when `res` has no `data` property', function(t) {
     var called_sanitisers = [];
 
     // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
@@ -26,7 +51,62 @@ module.exports.tests.sanitize = function(test, common) {
       t.deepEquals(called_sanitisers, expected_sanitisers);
       t.end();
     });
+
+  });
+
+  test('verify that all sanitisers were called as expected when res.data is empty', function(t) {
+    var called_sanitisers = [];
+
+    // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
+    //  were all called correctly
+    var search = proxyquire('../../../sanitiser/search_fallback', {
+      '../sanitiser/_text_autocomplete': function() {
+        called_sanitisers.push('_text_autocomplete');
+        return { errors: [], warnings: [] };
+      }
+    });
+
+    var expected_sanitisers = [
+      '_text_autocomplete'
+    ];
+
+    var req = {};
+    var res = {
+      data: []
+    };
+
+    search.middleware(req, res, function(){
+      t.deepEquals(called_sanitisers, expected_sanitisers);
+      t.end();
+    });
+
+  });
+
+  test('non-empty res.data should not call the _text_autocomplete sanitiser', function(t) {
+    var called_sanitisers = [];
+
+    // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
+    //  were all called correctly
+    var search = proxyquire('../../../sanitiser/search_fallback', {
+      '../sanitiser/_text_autocomplete': function() {
+        throw new Error('_text_autocomplete sanitiser should not have been called');
+      }
+    });
+
+    var expected_sanitisers = [];
+
+    var req = {};
+    var res = {
+      data: [{}]
+    };
+
+    search.middleware(req, res, function(){
+      t.deepEquals(called_sanitisers, expected_sanitisers);
+      t.end();
+    });
+
   });
+
 };
 
 module.exports.all = function (tape, common) {

From 46d58cafb04356b25b1fa1127c5ad9e66c9a7613 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 14:20:29 -0400
Subject: [PATCH 36/82] renamed `_text_autocomplete` to `_text_addressit`

---
 sanitiser/{_text_autocomplete.js => _text_addressit.js}         | 0
 sanitiser/autocomplete.js                                       | 2 +-
 test/unit/run.js                                                | 1 +
 .../sanitiser/{_text_autocomplete.js => _text_addressit.js}     | 2 +-
 4 files changed, 3 insertions(+), 2 deletions(-)
 rename sanitiser/{_text_autocomplete.js => _text_addressit.js} (100%)
 rename test/unit/sanitiser/{_text_autocomplete.js => _text_addressit.js} (98%)

diff --git a/sanitiser/_text_autocomplete.js b/sanitiser/_text_addressit.js
similarity index 100%
rename from sanitiser/_text_autocomplete.js
rename to sanitiser/_text_addressit.js
diff --git a/sanitiser/autocomplete.js b/sanitiser/autocomplete.js
index 025485cf..900edba2 100644
--- a/sanitiser/autocomplete.js
+++ b/sanitiser/autocomplete.js
@@ -3,7 +3,7 @@ var type_mapping = require('../helper/type_mapping');
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
-      text: require('../sanitiser/_text_autocomplete'),
+      text: require('../sanitiser/_text_addressit'),
       tokenizer: require('../sanitiser/_tokenizer'),
       size: require('../sanitiser/_size')(10, 10, 10),
       layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
diff --git a/test/unit/run.js b/test/unit/run.js
index bddb9ecf..77fb3415 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -51,6 +51,7 @@ var tests = [
   require('./sanitiser/_sources'),
   require('./sanitiser/_sources_and_layers'),
   require('./sanitiser/_text'),
+  require('./sanitiser/_text_addressit'),
   require('./sanitiser/_tokenizer'),
   require('./sanitiser/_deprecate_quattroshapes'),
   require('./sanitiser/_categories'),
diff --git a/test/unit/sanitiser/_text_autocomplete.js b/test/unit/sanitiser/_text_addressit.js
similarity index 98%
rename from test/unit/sanitiser/_text_autocomplete.js
rename to test/unit/sanitiser/_text_addressit.js
index a8911b69..6a67a3db 100644
--- a/test/unit/sanitiser/_text_autocomplete.js
+++ b/test/unit/sanitiser/_text_addressit.js
@@ -1,4 +1,4 @@
-var sanitiser = require('../../../sanitiser/_text_autocomplete');
+var sanitiser = require('../../../sanitiser/_text_addressit');
 var type_mapping = require('../../../helper/type_mapping');
 
 module.exports.tests = {};

From 484af8c04470b08638a204a92de4be82dc4a061b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 14:57:01 -0400
Subject: [PATCH 37/82] changed from `text_parser` to `text_parser_addressit`

this ensures that `clean.parsed_text` is correctly converted to query as expected
---
 query/autocomplete.js                         |   2 +-
 query/search_original.js                      |   2 +-
 ...tocomplete.js => text_parser_addressit.js} |   0
 test/unit/query/search_original.js            | 115 +++++++++---------
 4 files changed, 58 insertions(+), 61 deletions(-)
 rename query/{text_parser_autocomplete.js => text_parser_addressit.js} (100%)

diff --git a/query/autocomplete.js b/query/autocomplete.js
index 24e4d6ac..5f0826c0 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -1,7 +1,7 @@
 
 var peliasQuery = require('pelias-query'),
     defaults = require('./autocomplete_defaults'),
-    textParser = require('./text_parser_autocomplete'),
+    textParser = require('./text_parser_addressit'),
     check = require('check-types');
 
 // additional views (these may be merged in to pelias/query at a later date)
diff --git a/query/search_original.js b/query/search_original.js
index 5ab96248..4c923246 100644
--- a/query/search_original.js
+++ b/query/search_original.js
@@ -1,6 +1,6 @@
 var peliasQuery = require('pelias-query'),
     defaults = require('./search_defaults'),
-    textParser = require('./text_parser'),
+    textParser = require('./text_parser_addressit'),
     check = require('check-types');
 
 var placeTypes = require('../helper/placeTypes');
diff --git a/query/text_parser_autocomplete.js b/query/text_parser_addressit.js
similarity index 100%
rename from query/text_parser_autocomplete.js
rename to query/text_parser_addressit.js
diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js
index df0d2677..faf97286 100644
--- a/test/unit/query/search_original.js
+++ b/test/unit/query/search_original.js
@@ -86,65 +86,62 @@ module.exports.tests.query = function(test, common) {
     t.end();
   });
 
-  // test('valid query with a full valid address', function(t) {
-  //   var query = generate({ text: '123 main st new york ny 10010 US',
-  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-  //     querySize: 10,
-  //     parsed_text: {
-  //       number: '123',
-  //       street: 'main st',
-  //       state: 'NY',
-  //       country: 'USA',
-  //       postalcode: '10010'
-  //     }
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_full_address');
-  //
-  //   var fs = require('fs');
-  //   fs.writeFileSync('actual.json', JSON.stringify(compiled, null, 2));
-  //   fs.writeFileSync('expected.json', JSON.stringify(expected, null, 2));
-  //
-  //   t.deepEqual(compiled, expected, 'search_full_address');
-  //   t.end();
-  // });
-  //
-  // test('valid query with partial address', function(t) {
-  //   var query = generate({ text: 'soho grand, new york',
-  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-  //     querySize: 10,
-  //     parsed_text: { name: 'soho grand',
-  //       state: 'NY',
-  //       regions: [ 'soho grand' ],
-  //       admin_parts: 'new york'
-  //     }
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_partial_address');
-  //
-  //   t.deepEqual(compiled, expected, 'search_partial_address');
-  //   t.end();
-  // });
-  //
-  // test('valid query with regions in address', function(t) {
-  //   var query = generate({ text: '1 water st manhattan ny',
-  //     layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
-  //     querySize: 10,
-  //     parsed_text: { number: '1',
-  //       street: 'water st',
-  //       state: 'NY',
-  //       regions: [ 'manhattan' ]
-  //     },
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_regions_address');
-  //
-  //   t.deepEqual(compiled, expected, 'search_regions_address');
-  //   t.end();
-  // });
+  test('valid query with a full valid address', function(t) {
+    var query = generate({ text: '123 main st new york ny 10010 US',
+      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+      querySize: 10,
+      parsed_text: {
+        number: '123',
+        street: 'main st',
+        state: 'NY',
+        country: 'USA',
+        postalcode: '10010',
+        regions: [ 'new york' ]
+      }
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_full_address');
+
+    t.deepEqual(compiled, expected, 'search_full_address');
+    t.end();
+  });
+
+  test('valid query with partial address', function(t) {
+    var query = generate({ text: 'soho grand, new york',
+      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+      querySize: 10,
+      parsed_text: { name: 'soho grand',
+        state: 'NY',
+        regions: [ 'soho grand' ],
+        admin_parts: 'new york'
+      }
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_partial_address');
+
+    t.deepEqual(compiled, expected, 'search_partial_address');
+    t.end();
+  });
+
+  test('valid query with regions in address', function(t) {
+    var query = generate({ text: '1 water st manhattan ny',
+      layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ],
+      querySize: 10,
+      parsed_text: { number: '1',
+        street: 'water st',
+        state: 'NY',
+        regions: [ 'manhattan' ]
+      },
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_regions_address');
+
+    t.deepEqual(compiled, expected, 'search_regions_address');
+    t.end();
+  });
 
   test('valid boundary.country search', function(t) {
     var query = generate({

From 42affa80f8454e2e0074933bfcce28459e863c10 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 17 Aug 2016 15:21:53 -0400
Subject: [PATCH 38/82] updated filename

---
 sanitiser/search_fallback.js           |  2 +-
 test/unit/sanitiser/search_fallback.js | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/sanitiser/search_fallback.js b/sanitiser/search_fallback.js
index ebe6b53e..3a855cf9 100644
--- a/sanitiser/search_fallback.js
+++ b/sanitiser/search_fallback.js
@@ -1,6 +1,6 @@
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
-      text: require('../sanitiser/_text_autocomplete')
+      text: require('../sanitiser/_text_addressit')
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
diff --git a/test/unit/sanitiser/search_fallback.js b/test/unit/sanitiser/search_fallback.js
index 2003b19f..dd8fff2a 100644
--- a/test/unit/sanitiser/search_fallback.js
+++ b/test/unit/sanitiser/search_fallback.js
@@ -9,14 +9,14 @@ module.exports.tests.sanitize = function(test, common) {
     // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
     //  were all called correctly
     var search = proxyquire('../../../sanitiser/search_fallback', {
-      '../sanitiser/_text_autocomplete': function() {
-        called_sanitisers.push('_text_autocomplete');
+      '../sanitiser/_text_addressit': function() {
+        called_sanitisers.push('_text_addressit');
         return { errors: [], warnings: [] };
       }
     });
 
     var expected_sanitisers = [
-      '_text_autocomplete'
+      '_text_addressit'
     ];
 
     var req = {};
@@ -34,14 +34,14 @@ module.exports.tests.sanitize = function(test, common) {
     // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
     //  were all called correctly
     var search = proxyquire('../../../sanitiser/search_fallback', {
-      '../sanitiser/_text_autocomplete': function() {
-        called_sanitisers.push('_text_autocomplete');
+      '../sanitiser/_text_addressit': function() {
+        called_sanitisers.push('_text_addressit');
         return { errors: [], warnings: [] };
       }
     });
 
     var expected_sanitisers = [
-      '_text_autocomplete'
+      '_text_addressit'
     ];
 
     var req = {};
@@ -60,14 +60,14 @@ module.exports.tests.sanitize = function(test, common) {
     // rather than re-verify the functionality of all the sanitisers, this test just verifies that they
     //  were all called correctly
     var search = proxyquire('../../../sanitiser/search_fallback', {
-      '../sanitiser/_text_autocomplete': function() {
-        called_sanitisers.push('_text_autocomplete');
+      '../sanitiser/_text_addressit': function() {
+        called_sanitisers.push('_text_addressit');
         return { errors: [], warnings: [] };
       }
     });
 
     var expected_sanitisers = [
-      '_text_autocomplete'
+      '_text_addressit'
     ];
 
     var req = {};

From 165f6eadf1ea3d3996a6c24ec04464759e595285 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 19 Aug 2016 14:36:57 -0400
Subject: [PATCH 39/82] added logging in case where query required fallback

---
 sanitiser/search_fallback.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sanitiser/search_fallback.js b/sanitiser/search_fallback.js
index 3a855cf9..03773dd5 100644
--- a/sanitiser/search_fallback.js
+++ b/sanitiser/search_fallback.js
@@ -4,6 +4,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
     };
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
+var logger = require('pelias-logger').get('api:controller:search_fallback');
 
 // middleware
 module.exports.middleware = function( req, res, next ){
@@ -14,6 +15,11 @@ module.exports.middleware = function( req, res, next ){
     return next();
   }
 
+  // log the query that caused a fallback since libpostal+new-queries didn't return anything
+  if (req.path === '/v1/search') {
+    logger.info(req.clean.text);
+  }
+
   sanitize( req, function( err, clean ){
     next();
   });

From ae37add82be4b88e9e678a5865d597039a2ca947 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 19 Aug 2016 14:47:58 -0400
Subject: [PATCH 40/82] extracted variables to conform to convention

---
 routes/v1.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/routes/v1.js b/routes/v1.js
index 833240e9..4890a728 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -26,6 +26,11 @@ var controllers = {
   status: require('../controller/status')
 };
 
+var queries = {
+  libpostal: require('../query/search'),
+  fallback_to_old_prod: require('../query/search_original')
+};
+
 /** ----------------------- controllers ----------------------- **/
 
 var postProc = {
@@ -67,9 +72,9 @@ function addRoutes(app, peliasConfig) {
       // 2nd parameter is `backend` which gets initialized internally
       // 3rd parameter is which query module to use, use fallback/geodisambiguation
       //  first, then use original search strategy if first query didn't return anything
-      controllers.search(peliasConfig, undefined, require('../query/search')),
+      controllers.search(peliasConfig, undefined, queries.libpostal),
       sanitisers.search_fallback.middleware,
-      controllers.search(peliasConfig, undefined, require('../query/search_original')),
+      controllers.search(peliasConfig, undefined, queries.fallback_to_old_prod),
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),

From c8ba57f98b89346e61731550851e89debc4a871b Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 19 Aug 2016 15:18:42 -0400
Subject: [PATCH 41/82] added support for `dnt` flag when logging fallback
 query

---
 sanitiser/search_fallback.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sanitiser/search_fallback.js b/sanitiser/search_fallback.js
index 03773dd5..1782dbdb 100644
--- a/sanitiser/search_fallback.js
+++ b/sanitiser/search_fallback.js
@@ -5,6 +5,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
 
 var sanitize = function(req, cb) { sanitizeAll(req, sanitizers, cb); };
 var logger = require('pelias-logger').get('api:controller:search_fallback');
+var logging = require( '../helper/logging' );
 
 // middleware
 module.exports.middleware = function( req, res, next ){
@@ -17,7 +18,8 @@ module.exports.middleware = function( req, res, next ){
 
   // log the query that caused a fallback since libpostal+new-queries didn't return anything
   if (req.path === '/v1/search') {
-    logger.info(req.clean.text);
+    var queryText = logging.isDNT(req) ? '[text removed]' : req.clean.text;
+    logger.info(queryText);
   }
 
   sanitize( req, function( err, clean ){

From d68246809677a2fde085dc3834fe61b0329cfd36 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Sun, 21 Aug 2016 21:47:27 -0400
Subject: [PATCH 42/82] adapted tests to latest query module output

---
 test/unit/fixture/search_fallback.js          | 170 +++++++++++++++++-
 test/unit/fixture/search_geodisambiguation.js |  57 ++++++
 2 files changed, 220 insertions(+), 7 deletions(-)

diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
index 304841b5..96565e0e 100644
--- a/test/unit/fixture/search_fallback.js
+++ b/test/unit/fixture/search_fallback.js
@@ -47,6 +47,18 @@ module.exports = {
                   ]
                 }
               },
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
               {
                 'multi_match': {
                   'query': 'state value',
@@ -63,7 +75,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -121,6 +135,18 @@ module.exports = {
                   ]
                 }
               },
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
               {
                 'multi_match': {
                   'query': 'state value',
@@ -137,7 +163,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -185,6 +213,18 @@ module.exports = {
                   ]
                 }
               },
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
               {
                 'multi_match': {
                   'query': 'state value',
@@ -201,7 +241,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -239,6 +281,18 @@ module.exports = {
                   ]
                 }
               },
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
               {
                 'multi_match': {
                   'query': 'state value',
@@ -255,7 +309,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -281,6 +337,18 @@ module.exports = {
                   ]
                 }
               },
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
               {
                 'multi_match': {
                   'query': 'state value',
@@ -297,7 +365,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -309,6 +379,52 @@ module.exports = {
             }
           }
         },
+        {
+          'bool': {
+            '_name': 'fallback.county',
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'county value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.county',
+                    'parent.county_a',
+                    'parent.macrocounty',
+                    'parent.macrocounty_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'state value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.region',
+                    'parent.region_a'
+                  ]
+                }
+              },
+              {
+                'multi_match': {
+                  'query': 'country value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.country',
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'county'
+              }
+            }
+          }
+        },
         {
           'bool': {
             '_name': 'fallback.region',
@@ -329,7 +445,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -351,7 +469,9 @@ module.exports = {
                   'type': 'phrase',
                   'fields': [
                     'parent.country',
-                    'parent.country_a'
+                    'parent.country_a',
+                    'parent.dependency',
+                    'parent.dependency_a'
                   ]
                 }
               }
@@ -362,6 +482,42 @@ module.exports = {
               }
             }
           }
+        },
+        {
+          'function_score': {
+            'query': null,
+            'max_boost': 20,
+            'functions': [
+              {
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'popularity',
+                  'missing': 1
+                },
+                'weight': 1
+              }
+            ],
+            'score_mode': 'first',
+            'boost_mode': 'replace'
+          }
+        },
+        {
+          'function_score': {
+            'query': null,
+            'max_boost': 20,
+            'functions': [
+              {
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'population',
+                  'missing': 1
+                },
+                'weight': 2
+              }
+            ],
+            'score_mode': 'first',
+            'boost_mode': 'replace'
+          }
         }
       ]
     }
diff --git a/test/unit/fixture/search_geodisambiguation.js b/test/unit/fixture/search_geodisambiguation.js
index bd81e237..c0784252 100644
--- a/test/unit/fixture/search_geodisambiguation.js
+++ b/test/unit/fixture/search_geodisambiguation.js
@@ -170,6 +170,27 @@ module.exports = {
             }
           }
         },
+        {
+          'bool': {
+            'must': [
+              {
+                'multi_match': {
+                  'query': 'neighbourhood value',
+                  'type': 'phrase',
+                  'fields': [
+                    'parent.dependency',
+                    'parent.dependency_a'
+                  ]
+                }
+              }
+            ],
+            'filter': {
+              'term': {
+                'layer': 'dependency'
+              }
+            }
+          }
+        },
         {
           'bool': {
             'must': [
@@ -190,6 +211,42 @@ module.exports = {
               }
             }
           }
+        },
+        {
+          'function_score': {
+            'query': null,
+            'max_boost': 20,
+            'functions': [
+              {
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'popularity',
+                  'missing': 1
+                },
+                'weight': 1
+              }
+            ],
+            'score_mode': 'first',
+            'boost_mode': 'replace'
+          }
+        },
+        {
+          'function_score': {
+            'query': null,
+            'max_boost': 20,
+            'functions': [
+              {
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'population',
+                  'missing': 1
+                },
+                'weight': 2
+              }
+            ],
+            'score_mode': 'first',
+            'boost_mode': 'replace'
+          }
         }
       ]
     }

From 876782ed3a4c65386acfb058b4b2abf2f7ca51e0 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Tue, 30 Aug 2016 14:01:34 -0400
Subject: [PATCH 43/82] enabled all scores and filters

adjusted tests to account for new querying strategies, renamed all existing test fixture files to `*_original`.  pointed query module to a PR for testing.
---
 package.json                                  |    2 +-
 query/search.js                               |   53 +-
 test/unit/fixture/search_boundary_country.js  |  130 +--
 .../search_boundary_country_original.js       |   99 ++
 test/unit/fixture/search_fallback.js          | 1024 ++++++++---------
 ...ess.js => search_full_address_original.js} |    0
 test/unit/fixture/search_geodisambiguation.js |  480 ++++----
 test/unit/fixture/search_linguistic_bbox.js   |  134 +--
 .../search_linguistic_bbox_original.js        |   98 ++
 test/unit/fixture/search_linguistic_focus.js  |  154 +--
 .../fixture/search_linguistic_focus_bbox.js   |  174 +--
 .../search_linguistic_focus_bbox_original.js  |  128 +++
 .../search_linguistic_focus_null_island.js    |  154 +--
 ...h_linguistic_focus_null_island_original.js |  117 ++
 .../search_linguistic_focus_original.js       |  119 ++
 test/unit/fixture/search_linguistic_only.js   |  114 +-
 .../search_linguistic_only_original.js        |   89 ++
 .../fixture/search_linguistic_viewport.js     |  156 +--
 ...search_linguistic_viewport_min_diagonal.js |  159 +--
 ....js => search_partial_address_original.js} |    0
 ....js => search_regions_address_original.js} |    0
 .../fixture/search_with_category_filtering.js |  116 +-
 ...search_with_category_filtering_original.js |   86 ++
 .../fixture/search_with_source_filtering.js   |  112 +-
 .../search_with_source_filtering_original.js  |   85 ++
 test/unit/query/search.js                     |  295 ++---
 test/unit/query/search_original.js            |   22 +-
 27 files changed, 2204 insertions(+), 1896 deletions(-)
 create mode 100644 test/unit/fixture/search_boundary_country_original.js
 rename test/unit/fixture/{search_full_address.js => search_full_address_original.js} (100%)
 create mode 100644 test/unit/fixture/search_linguistic_bbox_original.js
 create mode 100644 test/unit/fixture/search_linguistic_focus_bbox_original.js
 create mode 100644 test/unit/fixture/search_linguistic_focus_null_island_original.js
 create mode 100644 test/unit/fixture/search_linguistic_focus_original.js
 create mode 100644 test/unit/fixture/search_linguistic_only_original.js
 rename test/unit/fixture/{search_partial_address.js => search_partial_address_original.js} (100%)
 rename test/unit/fixture/{search_regions_address.js => search_regions_address_original.js} (100%)
 create mode 100644 test/unit/fixture/search_with_category_filtering_original.js
 create mode 100644 test/unit/fixture/search_with_source_filtering_original.js

diff --git a/package.json b/package.json
index 2c4b940f..3753b785 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "8.5.0",
+    "pelias-query": "pelias/query#f890a72",
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"
diff --git a/query/search.js b/query/search.js
index 29c67876..b01fa6a1 100644
--- a/query/search.js
+++ b/query/search.js
@@ -7,43 +7,32 @@ var peliasQuery = require('pelias-query'),
 // general-purpose search query
 //------------------------------
 var fallbackQuery = new peliasQuery.layout.FallbackQuery();
-fallbackQuery.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
-fallbackQuery.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
-
 var geodisambiguationQuery = new peliasQuery.layout.GeodisambiguationQuery();
-geodisambiguationQuery.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
-geodisambiguationQuery.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
-
-// mandatory matches
-// query.score( peliasQuery.view.boundary_country, 'must' );
-// query.score( peliasQuery.view.ngrams, 'must' );
 
 // scoring boost
-// query.score( peliasQuery.view.phrase );
-// query.score( peliasQuery.view.focus( peliasQuery.view.phrase ) );
-// query.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) );
-// query.score( peliasQuery.view.population( peliasQuery.view.phrase ) );
-
-// address components
-// query.score( peliasQuery.view.address('housenumber'), 'must' );
-// query.score( peliasQuery.view.address('street'), 'must' );
-// query.score( peliasQuery.view.address('postcode'), 'must' );
-
-// admin components
-// country_a and region_a are left as matches here because the text-analyzer
-// can sometimes detect them, in which case a query more specific than a
-// multi_match is appropriate.
-// query.score( peliasQuery.view.admin('country_a'), 'must' );
-// query.score( peliasQuery.view.admin('region_a'), 'must' );
-// query.score( peliasQuery.view.admin('locality'), 'must' );
-// query.score( peliasQuery.view.admin_multi_match(adminFields, 'peliasAdmin') );
+fallbackQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
+fallbackQuery.score( peliasQuery.view.popularity_only_function );
+fallbackQuery.score( peliasQuery.view.population_only_function );
+
+geodisambiguationQuery.score( peliasQuery.view.focus_only_function( peliasQuery.view.phrase ) );
+geodisambiguationQuery.score( peliasQuery.view.popularity_only_function );
+geodisambiguationQuery.score( peliasQuery.view.population_only_function );
+// --------------------------------
 
 // non-scoring hard filters
-// query.filter( peliasQuery.view.boundary_circle );
-// query.filter( peliasQuery.view.boundary_rect );
-// query.filter( peliasQuery.view.sources );
-// query.filter( peliasQuery.view.layers );
-// query.filter( peliasQuery.view.categories );
+fallbackQuery.filter( peliasQuery.view.boundary_country );
+fallbackQuery.filter( peliasQuery.view.boundary_circle );
+fallbackQuery.filter( peliasQuery.view.boundary_rect );
+fallbackQuery.filter( peliasQuery.view.sources );
+fallbackQuery.filter( peliasQuery.view.layers );
+fallbackQuery.filter( peliasQuery.view.categories );
+
+geodisambiguationQuery.filter( peliasQuery.view.boundary_country );
+geodisambiguationQuery.filter( peliasQuery.view.boundary_circle );
+geodisambiguationQuery.filter( peliasQuery.view.boundary_rect );
+geodisambiguationQuery.filter( peliasQuery.view.sources );
+geodisambiguationQuery.filter( peliasQuery.view.layers );
+geodisambiguationQuery.filter( peliasQuery.view.categories );
 // --------------------------------
 
 /**
diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js
index 94f867b2..794301d7 100644
--- a/test/unit/fixture/search_boundary_country.js
+++ b/test/unit/fixture/search_boundary_country.js
@@ -1,99 +1,59 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [
-        {
-          'match': {
-            'parent.country_a': {
-              'analyzer': 'standard',
-              'query': 'ABC'
-            }
-          }
-        },
-        {
-          'match': {
-            'name.default': {
-              'query': 'test',
-              'boost': 1,
-              'analyzer': 'peliasQueryFullToken'
-            }
-          }
-        }
-      ],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      },{
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+            'bool': {
+              'should': []
             }
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'match': {
+                    'parent.country_a': {
+                      'analyzer': 'standard',
+                      'query': 'ABC'
+                    }
+                  }
+                },
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
+          }
         }
-      }],
-      'filter': [
+      },
+      'max_boost': 20,
+      'functions': [
         {
-          'terms': {
-            'layer': [
-              'test'
-            ]
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      ]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
 };
diff --git a/test/unit/fixture/search_boundary_country_original.js b/test/unit/fixture/search_boundary_country_original.js
new file mode 100644
index 00000000..94f867b2
--- /dev/null
+++ b/test/unit/fixture/search_boundary_country_original.js
@@ -0,0 +1,99 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [
+        {
+          'match': {
+            'parent.country_a': {
+              'analyzer': 'standard',
+              'query': 'ABC'
+            }
+          }
+        },
+        {
+          'match': {
+            'name.default': {
+              'query': 'test',
+              'boost': 1,
+              'analyzer': 'peliasQueryFullToken'
+            }
+          }
+        }
+      ],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
index 96565e0e..c5573018 100644
--- a/test/unit/fixture/search_fallback.js
+++ b/test/unit/fixture/search_fallback.js
@@ -1,525 +1,523 @@
 module.exports = {
   'query': {
-    'bool': {
-      'should': [
-        {
-          'bool': {
-            '_name': 'fallback.venue',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'query value',
-                  'type': 'phrase',
-                  'fields': [
-                    'phrase.default'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.neighbourhood',
-                    'parent.neighbourhood_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'borough value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.borough',
-                    'parent.borough_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'city value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a',
-                    'parent.localadmin',
-                    'parent.localadmin_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'venue'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.address',
-            'must': [
-              {
-                'match_phrase': {
-                  'address_parts.number': 'number value'
-                }
-              },
-              {
-                'match_phrase': {
-                  'address_parts.street': 'street value'
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.neighbourhood',
-                    'parent.neighbourhood_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'borough value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.borough',
-                    'parent.borough_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'city value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a',
-                    'parent.localadmin',
-                    'parent.localadmin_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'address'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.neighbourhood',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.neighbourhood',
-                    'parent.neighbourhood_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'borough value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.borough',
-                    'parent.borough_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'city value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a',
-                    'parent.localadmin',
-                    'parent.localadmin_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'neighbourhood'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.borough',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'borough value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.borough',
-                    'parent.borough_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'city value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a',
-                    'parent.localadmin',
-                    'parent.localadmin_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'borough'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.locality',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'city value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'locality'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.county',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'county value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a',
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'county'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.region',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'state value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              },
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'region'
-              }
+    'function_score': {
+      'query': {
+        'filtered': {
+          'query': {
+            'bool': {
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.venue',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'query value',
+                          'type': 'phrase',
+                          'fields': [
+                            'phrase.default'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.neighbourhood',
+                            'parent.neighbourhood_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'borough value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'venue'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.address',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.number': 'number value'
+                        }
+                      },
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.neighbourhood',
+                            'parent.neighbourhood_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'borough value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'address'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.neighbourhood',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.neighbourhood',
+                            'parent.neighbourhood_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'borough value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'neighbourhood'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.borough',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'borough value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'borough'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.locality',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'locality'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.county',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'county'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.region',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'region'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.country',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'country'
+                      }
+                    }
+                  }
+                }
+              ]
             }
-          }
-        },
-        {
-          'bool': {
-            '_name': 'fallback.country',
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'country value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a',
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'country'
-              }
+          },
+          'filter': {
+            'bool': {
+              'must': []
             }
           }
-        },
+        }
+      },
+      'max_boost': 20,
+      'functions': [
         {
-          'function_score': {
-            'query': null,
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'popularity',
-                  'missing': 1
-                },
-                'weight': 1
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
         },
         {
-          'function_score': {
-            'query': null,
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'population',
-                  'missing': 1
-                },
-                'weight': 2
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      ]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
   'size': 20,
diff --git a/test/unit/fixture/search_full_address.js b/test/unit/fixture/search_full_address_original.js
similarity index 100%
rename from test/unit/fixture/search_full_address.js
rename to test/unit/fixture/search_full_address_original.js
diff --git a/test/unit/fixture/search_geodisambiguation.js b/test/unit/fixture/search_geodisambiguation.js
index c0784252..d24fd0b7 100644
--- a/test/unit/fixture/search_geodisambiguation.js
+++ b/test/unit/fixture/search_geodisambiguation.js
@@ -1,254 +1,252 @@
 module.exports = {
   'query': {
-    'bool': {
-      'should': [
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.neighbourhood',
-                    'parent.neighbourhood_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'neighbourhood'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.borough',
-                    'parent.borough_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'borough'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.locality',
-                    'parent.locality_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'locality'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.localadmin',
-                    'parent.localadmin_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'localadmin'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.county',
-                    'parent.county_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'county'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.macrocounty',
-                    'parent.macrocounty_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'macrocounty'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.region',
-                    'parent.region_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'region'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.macroregion',
-                    'parent.macroregion_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'macroregion'
-              }
-            }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.dependency',
-                    'parent.dependency_a'
-                  ]
+    'function_score': {
+      'query': {
+        'filtered': {
+          'query': {
+            'bool': {
+              'should': [
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.neighbourhood',
+                            'parent.neighbourhood_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'neighbourhood'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'borough'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'locality'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'localadmin'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'county'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'macrocounty'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'region'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'macroregion'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'dependency'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'country'
+                      }
+                    }
+                  }
                 }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'dependency'
-              }
+              ]
             }
-          }
-        },
-        {
-          'bool': {
-            'must': [
-              {
-                'multi_match': {
-                  'query': 'neighbourhood value',
-                  'type': 'phrase',
-                  'fields': [
-                    'parent.country',
-                    'parent.country_a'
-                  ]
-                }
-              }
-            ],
-            'filter': {
-              'term': {
-                'layer': 'country'
-              }
+          },
+          'filter': {
+            'bool': {
+              'must': []
             }
           }
-        },
+        }
+      },
+      'max_boost': 20,
+      'functions': [
         {
-          'function_score': {
-            'query': null,
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'popularity',
-                  'missing': 1
-                },
-                'weight': 1
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
         },
         {
-          'function_score': {
-            'query': null,
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'population',
-                  'missing': 1
-                },
-                'weight': 2
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      ]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
   'size': 20,
diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js
index b8dbf3a1..670a73dd 100644
--- a/test/unit/fixture/search_linguistic_bbox.js
+++ b/test/unit/fixture/search_linguistic_bbox.js
@@ -1,98 +1,62 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      },{
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+            'bool': {
+              'should': []
             }
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
-            }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
-        }
-      }],
-      'filter': [{
-        'geo_bounding_box': {
-          'type': 'indexed',
-          'center_point': {
-            'top': 11.51,
-            'right': -61.84,
-            'bottom': 47.47,
-            'left': -103.16
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'geo_bounding_box': {
+                    'type': 'indexed',
+                    'center_point': {
+                      'top': 11.51,
+                      'right': -61.84,
+                      'bottom': 47.47,
+                      'left': -103.16
+                    }
+                  }
+                },
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
           }
+        }
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
         },
         {
-          'terms': {
-            'layer': [
-              'test'
-            ]
-          }
-      }]
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
+        }
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_bbox_original.js b/test/unit/fixture/search_linguistic_bbox_original.js
new file mode 100644
index 00000000..b8dbf3a1
--- /dev/null
+++ b/test/unit/fixture/search_linguistic_bbox_original.js
@@ -0,0 +1,98 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [{
+        'geo_bounding_box': {
+          'type': 'indexed',
+          'center_point': {
+            'top': 11.51,
+            'right': -61.84,
+            'bottom': 47.47,
+            'left': -103.16
+            }
+          }
+        },
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+      }]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js
index 38273273..477f5433 100644
--- a/test/unit/fixture/search_linguistic_focus.js
+++ b/test/unit/fixture/search_linguistic_focus.js
@@ -1,119 +1,65 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      }, {
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'boost': 1,
-                'slop': 2,
-                'query': 'test'
-              }
+            'bool': {
+              'should': []
             }
           },
-          'functions': [{
-            'linear': {
-              'center_point': {
-                'origin': {
-                  'lat': 29.49136,
-                  'lon': -82.50622
-                },
-                'offset': '0km',
-                'scale': '50km',
-                'decay': 0.5
-              }
-            },
-            'weight': 2
-          }],
-          'score_mode': 'avg',
-          'boost_mode': 'replace'
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
+          }
         }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'weight': 2,
+          'linear': {
+            'center_point': {
+              'origin': {
+                'lat': 29.49136,
+                'lon': -82.50622
+              },
+              'offset': '0km',
+              'scale': '50km',
+              'decay': 0.5
             }
+          }
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
-        }
-      }],
-      'filter': [
+          'weight': 1
+        },
         {
-          'terms': {
-            'layer': [
-              'test'
-            ]
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      ]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js
index ebc5f701..ef8a14a2 100644
--- a/test/unit/fixture/search_linguistic_focus_bbox.js
+++ b/test/unit/fixture/search_linguistic_focus_bbox.js
@@ -1,128 +1,76 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      }, {
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'boost': 1,
-                'slop': 2,
-                'query': 'test'
-              }
+            'bool': {
+              'should': []
             }
           },
-          'functions': [{
-            'linear': {
-              'center_point': {
-                'origin': {
-                  'lat': 29.49136,
-                  'lon': -82.50622
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'geo_bounding_box': {
+                    'type': 'indexed',
+                    'center_point': {
+                      'top': 11.51,
+                      'right': -61.84,
+                      'bottom': 47.47,
+                      'left': -103.16
+                    }
+                  }
                 },
-                'offset': '0km',
-                'scale': '50km',
-                'decay': 0.5
-              }
-            },
-            'weight': 2
-          }],
-          'score_mode': 'avg',
-          'boost_mode': 'replace'
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
-            }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
-        }
-      }],
-      'filter': [{
-        'geo_bounding_box': {
-          'type': 'indexed',
-          'center_point': {
-            'top': 11.51,
-            'right': -61.84,
-            'bottom': 47.47,
-            'left': -103.16
           }
         }
       },
-      {
-        'terms': {
-          'layer': [
-            'test'
-          ]
+      'max_boost': 20,
+      'functions': [
+        {
+          'weight': 2,
+          'linear': {
+            'center_point': {
+              'origin': {
+                'lat': 29.49136,
+                'lon': -82.50622
+              },
+              'offset': '0km',
+              'scale': '50km',
+              'decay': 0.5
+            }
+          }
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      }]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_focus_bbox_original.js b/test/unit/fixture/search_linguistic_focus_bbox_original.js
new file mode 100644
index 00000000..ebc5f701
--- /dev/null
+++ b/test/unit/fixture/search_linguistic_focus_bbox_original.js
@@ -0,0 +1,128 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'boost': 1,
+                'slop': 2,
+                'query': 'test'
+              }
+            }
+          },
+          'functions': [{
+            'linear': {
+              'center_point': {
+                'origin': {
+                  'lat': 29.49136,
+                  'lon': -82.50622
+                },
+                'offset': '0km',
+                'scale': '50km',
+                'decay': 0.5
+              }
+            },
+            'weight': 2
+          }],
+          'score_mode': 'avg',
+          'boost_mode': 'replace'
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [{
+        'geo_bounding_box': {
+          'type': 'indexed',
+          'center_point': {
+            'top': 11.51,
+            'right': -61.84,
+            'bottom': 47.47,
+            'left': -103.16
+          }
+        }
+      },
+      {
+        'terms': {
+          'layer': [
+            'test'
+          ]
+        }
+      }]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js
index 8f6fe381..962bd217 100644
--- a/test/unit/fixture/search_linguistic_focus_null_island.js
+++ b/test/unit/fixture/search_linguistic_focus_null_island.js
@@ -1,117 +1,65 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      }, {
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'boost': 1,
-                'slop': 2,
-                'query': 'test'
-              }
+            'bool': {
+              'should': []
             }
           },
-          'functions': [{
-            'linear': {
-              'center_point': {
-                'origin': {
-                  'lat': 0,
-                  'lon': 0
-                },
-                'offset': '0km',
-                'scale': '50km',
-                'decay': 0.5
-              }
-            },
-            'weight': 2
-          }],
-          'score_mode': 'avg',
-          'boost_mode': 'replace'
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
+          }
         }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'weight': 2,
+          'linear': {
+            'center_point': {
+              'origin': {
+                'lat': 0,
+                'lon': 0
+              },
+              'offset': '0km',
+              'scale': '50km',
+              'decay': 0.5
             }
+          }
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
-        }
-      }],
-      'filter':[{
-        'terms': {
-          'layer': [
-            'test'
-          ]
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      }]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_focus_null_island_original.js b/test/unit/fixture/search_linguistic_focus_null_island_original.js
new file mode 100644
index 00000000..8f6fe381
--- /dev/null
+++ b/test/unit/fixture/search_linguistic_focus_null_island_original.js
@@ -0,0 +1,117 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'boost': 1,
+                'slop': 2,
+                'query': 'test'
+              }
+            }
+          },
+          'functions': [{
+            'linear': {
+              'center_point': {
+                'origin': {
+                  'lat': 0,
+                  'lon': 0
+                },
+                'offset': '0km',
+                'scale': '50km',
+                'decay': 0.5
+              }
+            },
+            'weight': 2
+          }],
+          'score_mode': 'avg',
+          'boost_mode': 'replace'
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter':[{
+        'terms': {
+          'layer': [
+            'test'
+          ]
+        }
+      }]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_linguistic_focus_original.js b/test/unit/fixture/search_linguistic_focus_original.js
new file mode 100644
index 00000000..38273273
--- /dev/null
+++ b/test/unit/fixture/search_linguistic_focus_original.js
@@ -0,0 +1,119 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'boost': 1,
+                'slop': 2,
+                'query': 'test'
+              }
+            }
+          },
+          'functions': [{
+            'linear': {
+              'center_point': {
+                'origin': {
+                  'lat': 29.49136,
+                  'lon': -82.50622
+                },
+                'offset': '0km',
+                'scale': '50km',
+                'decay': 0.5
+              }
+            },
+            'weight': 2
+          }],
+          'score_mode': 'avg',
+          'boost_mode': 'replace'
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js
index 490eb0c9..1b181665 100644
--- a/test/unit/fixture/search_linguistic_only.js
+++ b/test/unit/fixture/search_linguistic_only.js
@@ -1,89 +1,51 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      },{
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+            'bool': {
+              'should': []
             }
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
+          }
         }
-      }],
-      'filter': [
+      },
+      'max_boost': 20,
+      'functions': [
         {
-          'terms': {
-            'layer': [
-              'test'
-            ]
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      ]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 10,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_only_original.js b/test/unit/fixture/search_linguistic_only_original.js
new file mode 100644
index 00000000..490eb0c9
--- /dev/null
+++ b/test/unit/fixture/search_linguistic_only_original.js
@@ -0,0 +1,89 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [
+        {
+          'terms': {
+            'layer': [
+              'test'
+            ]
+          }
+        }
+      ]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 10,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js
index ca6414a7..1b181665 100644
--- a/test/unit/fixture/search_linguistic_viewport.js
+++ b/test/unit/fixture/search_linguistic_viewport.js
@@ -1,133 +1,51 @@
 module.exports = {
   'query': {
-    'bool': {
-      'must': [
-        {
-          'match': {
-            'name.default': {
-              'analyzer': 'peliasQueryFullToken',
-              'boost': 1,
-              'query': 'test'
-            }
-          }
-        }
-      ],
-      'should': [
-        {
-          'match': {
-            'phrase.default': {
-              'analyzer': 'peliasPhrase',
-              'type': 'phrase',
-              'boost': 1,
-              'slop': 2,
-              'query': 'test'
+    'function_score': {
+      'query': {
+        'filtered': {
+          'query': {
+            'bool': {
+              'should': []
             }
-          }
-        },
-        {
-          'function_score': {
-            'query': {
-              'match': {
-                'phrase.default': {
-                  'analyzer': 'peliasPhrase',
-                  'type': 'phrase',
-                  'boost': 1,
-                  'slop': 2,
-                  'query': 'test'
-                }
-              }
-            },
-            'functions': [
-              {
-                'weight': 2,
-                'linear': {
-                  'center_point': {
-                    'origin': {
-                      'lat': 29.49136,
-                      'lon': -82.50622
-                    },
-                    'offset': '0km',
-                    'scale': '50km',
-                    'decay': 0.5
+          },
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
                   }
                 }
-              }
-            ],
-            'score_mode': 'avg',
-            'boost_mode': 'replace'
+              ]
+            }
           }
-        },
+        }
+      },
+      'max_boost': 20,
+      'functions': [
         {
-          'function_score': {
-            'query': {
-              'match': {
-                'phrase.default': {
-                  'analyzer': 'peliasPhrase',
-                  'type': 'phrase',
-                  'boost': 1,
-                  'slop': 2,
-                  'query': 'test'
-                }
-              }
-            },
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'popularity',
-                  'missing': 1
-                },
-                'weight': 1
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
         },
         {
-          'function_score': {
-            'query': {
-              'match': {
-                'phrase.default': {
-                  'analyzer': 'peliasPhrase',
-                  'type': 'phrase',
-                  'boost': 1,
-                  'slop': 2,
-                  'query': 'test'
-                }
-              }
-            },
-            'max_boost': 20,
-            'functions': [
-              {
-                'field_value_factor': {
-                  'modifier': 'log1p',
-                  'field': 'population',
-                  'missing': 1
-                },
-                'weight': 2
-              }
-            ],
-            'score_mode': 'first',
-            'boost_mode': 'replace'
-          }
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
       ],
-      'filter': [
-        {
-          'terms': {
-            'layer': [
-              'test'
-            ]
-          }
-        }
-      ]
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
   'size': 10,
-  'track_scores': true,
-  'sort': [
-    '_score'
-  ]
-};
+  'track_scores': true
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
index e5dbb862..1b181665 100644
--- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
+++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
@@ -1,128 +1,51 @@
 module.exports = {
   'query': {
-    'filtered': {
+    'function_score': {
       'query': {
-        'bool': {
-          'must': [
-            {
-              'match': {
-                'name.default': {
-                  'analyzer': 'peliasQueryFullToken',
-                  'boost': 1,
-                  'query': 'test'
-                }
-              }
+        'filtered': {
+          'query': {
+            'bool': {
+              'should': []
             }
-          ],
-          'should': [
-            {
-              'match': {
-                'phrase.default': {
-                  'analyzer': 'peliasPhrase',
-                  'type': 'phrase',
-                  'boost': 1,
-                  'slop': 2,
-                  'query': 'test'
-                }
-              }
-            },
-            {
-              'function_score': {
-                'query': {
-                  'match': {
-                    'phrase.default': {
-                      'analyzer': 'peliasPhrase',
-                      'type': 'phrase',
-                      'boost': 1,
-                      'slop': 2,
-                      'query': 'test'
-                    }
-                  }
-                },
-                'functions': [
-                  {
-                    'weight': 2,
-                    'linear': {
-                      'center_point': {
-                        'origin': {
-                          'lat': 28.49136,
-                          'lon': -87.50623
-                        },
-                        'offset': '0km',
-                        'scale': '1km',
-                        'decay': 0.5
-                      }
-                    }
-                  }
-                ],
-                'score_mode': 'avg',
-                'boost_mode': 'replace'
-              }
-            },
-            {
-              'function_score': {
-                'query': {
-                  'match': {
-                    'phrase.default': {
-                      'analyzer': 'peliasPhrase',
-                      'type': 'phrase',
-                      'boost': 1,
-                      'slop': 2,
-                      'query': 'test'
-                    }
-                  }
-                },
-                'max_boost': 20,
-                'functions': [
-                  {
-                    'field_value_factor': {
-                      'modifier': 'log1p',
-                      'field': 'popularity',
-                      'missing': 1
-                    },
-                    'weight': 1
+          },
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'layer': [
+                      'test'
+                    ]
                   }
-                ],
-                'score_mode': 'first',
-                'boost_mode': 'replace'
-              }
-            },
-            {
-              'function_score': {
-                'query': {
-                  'match': {
-                    'phrase.default': {
-                      'analyzer': 'peliasPhrase',
-                      'type': 'phrase',
-                      'boost': 1,
-                      'slop': 2,
-                      'query': 'test'
-                    }
-                  }
-                },
-                'max_boost': 20,
-                'functions': [
-                  {
-                    'field_value_factor': {
-                      'modifier': 'log1p',
-                      'field': 'population',
-                      'missing': 1
-                    },
-                    'weight': 2
-                  }
-                ],
-                'score_mode': 'first',
-                'boost_mode': 'replace'
-              }
+                }
+              ]
             }
-          ]
+          }
+        }
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      }
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
   'size': 10,
-  'track_scores': true,
-  'sort': [
-    '_score'
-  ]
-};
+  'track_scores': true
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_partial_address.js b/test/unit/fixture/search_partial_address_original.js
similarity index 100%
rename from test/unit/fixture/search_partial_address.js
rename to test/unit/fixture/search_partial_address_original.js
diff --git a/test/unit/fixture/search_regions_address.js b/test/unit/fixture/search_regions_address_original.js
similarity index 100%
rename from test/unit/fixture/search_regions_address.js
rename to test/unit/fixture/search_regions_address_original.js
diff --git a/test/unit/fixture/search_with_category_filtering.js b/test/unit/fixture/search_with_category_filtering.js
index ca1f26bb..a13b4852 100644
--- a/test/unit/fixture/search_with_category_filtering.js
+++ b/test/unit/fixture/search_with_category_filtering.js
@@ -1,86 +1,52 @@
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      }, {
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+            'bool': {
+              'should': []
             }
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      }, {
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'category': [
+                      'retail',
+                      'food'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
+          }
         }
-      }],
-      'filter': [{
-        'terms': {
-          'category': ['retail', 'food']
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      }]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
   'size': 20,
-  'track_scores': true,
-  'sort': [
-    '_score'
-  ]
-};
+  'track_scores': true
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_with_category_filtering_original.js b/test/unit/fixture/search_with_category_filtering_original.js
new file mode 100644
index 00000000..ca1f26bb
--- /dev/null
+++ b/test/unit/fixture/search_with_category_filtering_original.js
@@ -0,0 +1,86 @@
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      }, {
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [{
+        'terms': {
+          'category': ['retail', 'food']
+        }
+      }]
+    }
+  },
+  'size': 20,
+  'track_scores': true,
+  'sort': [
+    '_score'
+  ]
+};
diff --git a/test/unit/fixture/search_with_source_filtering.js b/test/unit/fixture/search_with_source_filtering.js
index 24da9468..2ae9346b 100644
--- a/test/unit/fixture/search_with_source_filtering.js
+++ b/test/unit/fixture/search_with_source_filtering.js
@@ -1,85 +1,51 @@
-
 module.exports = {
   'query': {
-    'bool': {
-      'must': [{
-        'match': {
-          'name.default': {
-            'query': 'test',
-            'boost': 1,
-            'analyzer': 'peliasQueryFullToken'
-          }
-        }
-      }],
-      'should': [{
-        'match': {
-          'phrase.default': {
-            'query': 'test',
-            'analyzer': 'peliasPhrase',
-            'type': 'phrase',
-            'boost': 1,
-            'slop': 2
-          }
-        }
-      },{
-        'function_score': {
+    'function_score': {
+      'query': {
+        'filtered': {
           'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+            'bool': {
+              'should': []
             }
           },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'popularity',
-              'missing': 1
-            },
-            'weight': 1
-          }]
-        }
-      },{
-        'function_score': {
-          'query': {
-            'match': {
-              'phrase.default': {
-                'query': 'test',
-                'analyzer': 'peliasPhrase',
-                'type': 'phrase',
-                'slop': 2,
-                'boost': 1
-              }
+          'filter': {
+            'bool': {
+              'must': [
+                {
+                  'terms': {
+                    'source': [
+                      'test_source'
+                    ]
+                  }
+                }
+              ]
             }
-          },
-          'max_boost': 20,
-          'score_mode': 'first',
-          'boost_mode': 'replace',
-          'functions': [{
-            'field_value_factor': {
-              'modifier': 'log1p',
-              'field': 'population',
-              'missing': 1
-            },
-            'weight': 2
-          }]
+          }
         }
-      }],
-      'filter': [{
-        'terms': {
-          'source': ['test_source']
+      },
+      'max_boost': 20,
+      'functions': [
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'popularity',
+            'missing': 1
+          },
+          'weight': 1
+        },
+        {
+          'field_value_factor': {
+            'modifier': 'log1p',
+            'field': 'population',
+            'missing': 1
+          },
+          'weight': 2
         }
-      }]
+      ],
+      'score_mode': 'avg',
+      'boost_mode': 'replace'
     }
   },
-  'sort': [ '_score' ],
   'size': 20,
   'track_scores': true
-};
+};
\ No newline at end of file
diff --git a/test/unit/fixture/search_with_source_filtering_original.js b/test/unit/fixture/search_with_source_filtering_original.js
new file mode 100644
index 00000000..24da9468
--- /dev/null
+++ b/test/unit/fixture/search_with_source_filtering_original.js
@@ -0,0 +1,85 @@
+
+module.exports = {
+  'query': {
+    'bool': {
+      'must': [{
+        'match': {
+          'name.default': {
+            'query': 'test',
+            'boost': 1,
+            'analyzer': 'peliasQueryFullToken'
+          }
+        }
+      }],
+      'should': [{
+        'match': {
+          'phrase.default': {
+            'query': 'test',
+            'analyzer': 'peliasPhrase',
+            'type': 'phrase',
+            'boost': 1,
+            'slop': 2
+          }
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'popularity',
+              'missing': 1
+            },
+            'weight': 1
+          }]
+        }
+      },{
+        'function_score': {
+          'query': {
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'slop': 2,
+                'boost': 1
+              }
+            }
+          },
+          'max_boost': 20,
+          'score_mode': 'first',
+          'boost_mode': 'replace',
+          'functions': [{
+            'field_value_factor': {
+              'modifier': 'log1p',
+              'field': 'population',
+              'missing': 1
+            },
+            'weight': 2
+          }]
+        }
+      }],
+      'filter': [{
+        'terms': {
+          'source': ['test_source']
+        }
+      }]
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 20,
+  'track_scores': true
+};
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index 2f7fa951..86e582b7 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -1,4 +1,5 @@
 var generate = require('../../../query/search');
+var fs = require('fs');
 
 module.exports.tests = {};
 
@@ -10,117 +11,117 @@ module.exports.tests.interface = function(test, common) {
 };
 
 module.exports.tests.query = function(test, common) {
-  // test('valid search + focus + bbox', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
-  //     'boundary.rect.min_lat': 47.47,
-  //     'boundary.rect.max_lon': -61.84,
-  //     'boundary.rect.max_lat': 11.51,
-  //     'boundary.rect.min_lon': -103.16,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_focus_bbox');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
-  //   t.end();
-  // });
-
-  // test('valid search + bbox', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'boundary.rect.min_lat': 47.47,
-  //     'boundary.rect.max_lon': -61.84,
-  //     'boundary.rect.max_lat': 11.51,
-  //     'boundary.rect.min_lon': -103.16,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_bbox');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_bbox');
-  //   t.end();
-  // });
-
-  // test('valid lingustic-only search', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_only');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_only');
-  //   t.end();
-  // });
-
-  // test('search search + focus', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_focus');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_focus');
-  //   t.end();
-  // });
-
-  // test('search search + viewport', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'focus.viewport.min_lat': 28.49136,
-  //     'focus.viewport.max_lat': 30.49136,
-  //     'focus.viewport.min_lon': -87.50622,
-  //     'focus.viewport.max_lon': -77.50622,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_viewport');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_viewport');
-  //   t.end();
-  // });
+  test('valid search + focus + bbox', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+      'boundary.rect.min_lat': 47.47,
+      'boundary.rect.max_lon': -61.84,
+      'boundary.rect.max_lat': 11.51,
+      'boundary.rect.min_lon': -103.16,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus_bbox');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
+    t.end();
+  });
+
+  test('valid search + bbox', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'boundary.rect.min_lat': 47.47,
+      'boundary.rect.max_lon': -61.84,
+      'boundary.rect.max_lat': 11.51,
+      'boundary.rect.min_lon': -103.16,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_bbox');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_bbox');
+    t.end();
+  });
+
+  test('valid lingustic-only search', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_only');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_only');
+    t.end();
+  });
+
+  test('search search + focus', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus');
+    t.end();
+  });
+
+  test('search search + viewport', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.viewport.min_lat': 28.49136,
+      'focus.viewport.max_lat': 30.49136,
+      'focus.viewport.min_lon': -87.50622,
+      'focus.viewport.max_lon': -77.50622,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_viewport');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_viewport');
+    t.end();
+  });
 
   // viewport scale sizing currently disabled.
   // ref: https://github.com/pelias/api/pull/388
-  // test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'focus.viewport.min_lat': 28.49135,
-  //     'focus.viewport.max_lat': 28.49137,
-  //     'focus.viewport.min_lon': -87.50622,
-  //     'focus.viewport.max_lon': -87.50624,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
-  //
-  //   t.deepEqual(compiled, expected, 'valid search query');
-  //   t.end();
-  // });
-
-  // test('search search + focus on null island', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     'focus.point.lat': 0, 'focus.point.lon': 0,
-  //     layers: ['test']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_linguistic_focus_null_island');
-  //
-  //   t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
-  //   t.end();
-  // });
+  test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.viewport.min_lat': 28.49135,
+      'focus.viewport.max_lat': 28.49137,
+      'focus.viewport.min_lon': -87.50622,
+      'focus.viewport.max_lon': -87.50624,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
+
+    t.deepEqual(compiled, expected, 'valid search query');
+    t.end();
+  });
+
+  test('search search + focus on null island', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      'focus.point.lat': 0, 'focus.point.lon': 0,
+      layers: ['test']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_linguistic_focus_null_island');
+
+    t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
+    t.end();
+  });
 
   test('parsed_text with all fields should use FallbackQuery', function(t) {
     var clean = {
@@ -166,45 +167,45 @@ module.exports.tests.query = function(test, common) {
 
   });
 
-  // test('valid boundary.country search', function(t) {
-  //   var query = generate({
-  //     text: 'test', querySize: 10,
-  //     layers: ['test'],
-  //     'boundary.country': 'ABC'
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_boundary_country');
-  //
-  //   t.deepEqual(compiled, expected, 'search: valid boundary.country query');
-  //   t.end();
-  // });
-
-  // test('valid sources filter', function(t) {
-  //   var query = generate({
-  //     'text': 'test',
-  //     'sources': ['test_source']
-  //   });
-  //
-  //   var compiled = JSON.parse( JSON.stringify( query ) );
-  //   var expected = require('../fixture/search_with_source_filtering');
-  //
-  //   t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
-  //   t.end();
-  // });
-
-  //test('categories filter', function(t) {
-    //var query = generate({
-      //'text': 'test',
-      //'categories': ['retail','food']
-    //});
-
-    //var compiled = JSON.parse( JSON.stringify( query ) );
-    //var expected = require('../fixture/search_with_category_filtering');
-
-    //t.deepEqual(compiled, expected, 'valid search query with category filtering');
-    //t.end();
-  //});
+  test('valid boundary.country search', function(t) {
+    var query = generate({
+      text: 'test', querySize: 10,
+      layers: ['test'],
+      'boundary.country': 'ABC'
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_boundary_country');
+
+    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
+    t.end();
+  });
+
+  test('valid sources filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'sources': ['test_source']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_source_filtering');
+
+    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
+    t.end();
+  });
+
+  test('categories filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'categories': ['retail','food']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_category_filtering');
+
+    t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {
diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js
index faf97286..b66bbca9 100644
--- a/test/unit/query/search_original.js
+++ b/test/unit/query/search_original.js
@@ -22,7 +22,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus_bbox');
+    var expected = require('../fixture/search_linguistic_focus_bbox_original');
 
     t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
     t.end();
@@ -39,7 +39,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_bbox');
+    var expected = require('../fixture/search_linguistic_bbox_original');
 
     t.deepEqual(compiled, expected, 'search_linguistic_bbox');
     t.end();
@@ -52,7 +52,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_only');
+    var expected = require('../fixture/search_linguistic_only_original');
 
     t.deepEqual(compiled, expected, 'search_linguistic_only');
     t.end();
@@ -66,7 +66,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus');
+    var expected = require('../fixture/search_linguistic_focus_original');
 
     t.deepEqual(compiled, expected, 'search_linguistic_focus');
     t.end();
@@ -80,7 +80,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_focus_null_island');
+    var expected = require('../fixture/search_linguistic_focus_null_island_original');
 
     t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
     t.end();
@@ -101,7 +101,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_full_address');
+    var expected = require('../fixture/search_full_address_original');
 
     t.deepEqual(compiled, expected, 'search_full_address');
     t.end();
@@ -119,7 +119,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_partial_address');
+    var expected = require('../fixture/search_partial_address_original');
 
     t.deepEqual(compiled, expected, 'search_partial_address');
     t.end();
@@ -137,7 +137,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_regions_address');
+    var expected = require('../fixture/search_regions_address_original');
 
     t.deepEqual(compiled, expected, 'search_regions_address');
     t.end();
@@ -151,7 +151,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_boundary_country');
+    var expected = require('../fixture/search_boundary_country_original');
 
     t.deepEqual(compiled, expected, 'search: valid boundary.country query');
     t.end();
@@ -164,7 +164,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_with_source_filtering');
+    var expected = require('../fixture/search_with_source_filtering_original');
 
     t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
     t.end();
@@ -177,7 +177,7 @@ module.exports.tests.query = function(test, common) {
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_with_category_filtering');
+    var expected = require('../fixture/search_with_category_filtering_original');
 
     t.deepEqual(compiled, expected, 'valid search query with category filtering');
     t.end();

From 8339a66e91fb617e816eeaf9f22e61690e2b5d1a Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 31 Aug 2016 15:15:40 -0400
Subject: [PATCH 44/82] updated query module to branch from hashcode fixed
 tests to use `boost_mode` value of `multiply` instead of `replace`

---
 package.json                                                 | 2 +-
 test/unit/fixture/search_boundary_country.js                 | 2 +-
 test/unit/fixture/search_fallback.js                         | 2 +-
 test/unit/fixture/search_geodisambiguation.js                | 2 +-
 test/unit/fixture/search_linguistic_bbox.js                  | 4 ++--
 test/unit/fixture/search_linguistic_focus.js                 | 4 ++--
 test/unit/fixture/search_linguistic_focus_bbox.js            | 4 ++--
 test/unit/fixture/search_linguistic_focus_null_island.js     | 4 ++--
 test/unit/fixture/search_linguistic_only.js                  | 4 ++--
 test/unit/fixture/search_linguistic_viewport.js              | 4 ++--
 test/unit/fixture/search_linguistic_viewport_min_diagonal.js | 4 ++--
 test/unit/fixture/search_with_category_filtering.js          | 4 ++--
 test/unit/fixture/search_with_source_filtering.js            | 4 ++--
 13 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/package.json b/package.json
index 3753b785..8c88388c 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "pelias/query#f890a72",
+    "pelias-query": "pelias/query#reorganize-queries-for-scoring",
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"
diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js
index 794301d7..977b15c9 100644
--- a/test/unit/fixture/search_boundary_country.js
+++ b/test/unit/fixture/search_boundary_country.js
@@ -51,7 +51,7 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
index c5573018..6c076c61 100644
--- a/test/unit/fixture/search_fallback.js
+++ b/test/unit/fixture/search_fallback.js
@@ -517,7 +517,7 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 20,
diff --git a/test/unit/fixture/search_geodisambiguation.js b/test/unit/fixture/search_geodisambiguation.js
index d24fd0b7..34b447e1 100644
--- a/test/unit/fixture/search_geodisambiguation.js
+++ b/test/unit/fixture/search_geodisambiguation.js
@@ -246,7 +246,7 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 20,
diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js
index 670a73dd..9470529d 100644
--- a/test/unit/fixture/search_linguistic_bbox.js
+++ b/test/unit/fixture/search_linguistic_bbox.js
@@ -54,9 +54,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js
index 477f5433..25cefa2e 100644
--- a/test/unit/fixture/search_linguistic_focus.js
+++ b/test/unit/fixture/search_linguistic_focus.js
@@ -57,9 +57,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js
index ef8a14a2..6523232f 100644
--- a/test/unit/fixture/search_linguistic_focus_bbox.js
+++ b/test/unit/fixture/search_linguistic_focus_bbox.js
@@ -68,9 +68,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js
index 962bd217..c6ccdd40 100644
--- a/test/unit/fixture/search_linguistic_focus_null_island.js
+++ b/test/unit/fixture/search_linguistic_focus_null_island.js
@@ -57,9 +57,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js
index 1b181665..08254737 100644
--- a/test/unit/fixture/search_linguistic_only.js
+++ b/test/unit/fixture/search_linguistic_only.js
@@ -43,9 +43,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js
index 1b181665..08254737 100644
--- a/test/unit/fixture/search_linguistic_viewport.js
+++ b/test/unit/fixture/search_linguistic_viewport.js
@@ -43,9 +43,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
index 1b181665..08254737 100644
--- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
+++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
@@ -43,9 +43,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 10,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_with_category_filtering.js b/test/unit/fixture/search_with_category_filtering.js
index a13b4852..3c217c35 100644
--- a/test/unit/fixture/search_with_category_filtering.js
+++ b/test/unit/fixture/search_with_category_filtering.js
@@ -44,9 +44,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 20,
   'track_scores': true
-};
\ No newline at end of file
+};
diff --git a/test/unit/fixture/search_with_source_filtering.js b/test/unit/fixture/search_with_source_filtering.js
index 2ae9346b..454c317d 100644
--- a/test/unit/fixture/search_with_source_filtering.js
+++ b/test/unit/fixture/search_with_source_filtering.js
@@ -43,9 +43,9 @@ module.exports = {
         }
       ],
       'score_mode': 'avg',
-      'boost_mode': 'replace'
+      'boost_mode': 'multiply'
     }
   },
   'size': 20,
   'track_scores': true
-};
\ No newline at end of file
+};

From a8e82b018dd76c2dee3886ff815b4f7c5471f79c Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Sat, 3 Sep 2016 14:20:42 -0400
Subject: [PATCH 45/82] Refactor deduper and write additional tests

---
 helper/diffPlaces.js                          |  172 +++
 middleware/dedupe.js                          |   88 +-
 .../fixture/dedupe_elasticsearch_results.js   | 1261 ++++++++++++++---
 test/unit/helper/diffPlaces.js                |  180 +++
 test/unit/middleware/dedupe.js                |    2 +-
 test/unit/run.js                              |    1 +
 6 files changed, 1394 insertions(+), 310 deletions(-)
 create mode 100644 helper/diffPlaces.js
 create mode 100644 test/unit/helper/diffPlaces.js

diff --git a/helper/diffPlaces.js b/helper/diffPlaces.js
new file mode 100644
index 00000000..e6a48af5
--- /dev/null
+++ b/helper/diffPlaces.js
@@ -0,0 +1,172 @@
+var _ = require('lodash');
+var placeTypes = require('../helper/placeTypes');
+
+/**
+ * Compare the layer properties if they exist.
+ * Returns false if the objects are the same, and throws
+ * an exception with the message 'different' if not.
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDiffLayer(item1, item2) {
+  if (item1.layer === item2.layer) {
+    return false;
+  }
+
+  throw new Error('different');
+}
+
+/**
+ * Compare the parent.* properties if they exist.
+ * Returns false if the objects are the same, and throws
+ * an exception with the message 'different' if not.
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDiffParentHierarchy(item1, item2) {
+  // if neither object has parent, assume same
+  if (!item1.hasOwnProperty('parent') && !item2.hasOwnProperty('parent')) {
+    return false;
+  }
+
+  // if both have parent, do the rest of the checking
+  if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) {
+    placeTypes.forEach(function (placeType) {
+      // don't consider its own id
+      if (placeType === item1.layer) {
+        return;
+      }
+      propMatch(item1.parent, item2.parent, placeType + '_id');
+    });
+    return false;
+  }
+
+  // if one has parent and the other doesn't consider different
+  throw new Error('different');
+}
+
+/**
+ * Compare the name.* properties if they exist.
+ * Returns false if the objects are the same, and throws
+ * an exception with the message 'different' if not.
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDiffName(item1, item2) {
+  if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
+    for (var lang in item1.name) {
+      if(item2.name[lang] || lang === 'default') {
+        // do not consider absence of an additional name as a difference
+        propMatch(item1.name, item2.name, lang);
+      }
+    }
+  }
+  else {
+    propMatch(item1, item2, 'name');
+  }
+}
+
+/**
+ * Compare the address_parts properties if they exist.
+ * Returns false if the objects are the same, and throws
+ * an exception with the message 'different' if not.
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDiffAddress(item1, item2) {
+  // if neither record has address, assume same
+  if (!item1.hasOwnProperty('address_parts') && !item2.hasOwnProperty('address_parts')) {
+    return false;
+  }
+
+  // if both have address, check parts
+  if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) {
+    propMatch(item1.address_parts, item2.address_parts, 'number');
+    propMatch(item1.address_parts, item2.address_parts, 'street');
+
+    // only compare zip if both records have it, otherwise just ignore and assume it's the same
+    // since by this time we've already compared parent hierarchies
+    if (item1.address_parts.hasOwnProperty('zip') && item2.address_parts.hasOwnProperty('zip')) {
+      propMatch(item1.address_parts, item2.address_parts, 'zip');
+    }
+
+    return false;
+  }
+
+  // one has address and the other doesn't, different!
+  throw new Error('different');
+}
+
+/**
+ * Compare the two records and return true if they differ and false if same.
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDifferent(item1, item2) {
+  try {
+    isDiffLayer(item1, item2);
+    isDiffParentHierarchy(item1, item2);
+    isDiffName(item1, item2);
+    isDiffAddress(item1, item2);
+  }
+  catch (err) {
+    if (err.message === 'different') {
+      return true;
+    }
+    throw err;
+  }
+
+  return false;
+}
+
+/**
+ * Throw exception if properties are different
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @param {string} prop
+ * @throws {Error}
+ */
+function propMatch(item1, item2, prop) {
+  var prop1 = item1[prop];
+  var prop2 = item2[prop];
+
+  // in the case the property is an array (currently only in parent schema)
+  // simply take the 1st item. this will change in the near future to support multiple hierarchies
+  if (_.isArray(prop1)) { prop1 = prop1[0]; }
+  if (_.isArray(prop2)) { prop2 = prop2[0]; }
+
+  if (normalizeString(prop1) !== normalizeString(prop2)) {
+    throw new Error('different');
+  }
+}
+
+/**
+ * Remove punctuation and lowercase
+ *
+ * @param {string} str
+ * @returns {string}
+ */
+function normalizeString(str) {
+  if (!str) {
+    return '';
+  }
+  return str.toLowerCase().split(/[ ,-]+/).join(' ');
+}
+
+module.exports.isDifferent = isDifferent;
\ No newline at end of file
diff --git a/middleware/dedupe.js b/middleware/dedupe.js
index 786675c4..b40f1806 100644
--- a/middleware/dedupe.js
+++ b/middleware/dedupe.js
@@ -1,5 +1,6 @@
 var logger = require('pelias-logger').get('api');
 var _ = require('lodash');
+var isDifferent = require('../helper/diffPlaces').isDifferent;
 
 function setup() {
   return dedupeResults;
@@ -19,7 +20,7 @@ function dedupeResults(req, res, next) {
       uniqueResults.push(hit);
     }
     else {
-      logger.info('[dupe]', { query: req.clean.text, hit: hit.name.default });
+      logger.info('[dupe]', { query: req.clean.text, hit: hit.name.default + ' ' + hit.source + ':' + hit._id });
     }
 
     // stop looping when requested size has been reached in uniqueResults
@@ -31,89 +32,4 @@ function dedupeResults(req, res, next) {
   next();
 }
 
-/**
- * @param {object} item1
- * @param {object} item2
- * @returns {boolean}
- * @throws {Error}
- */
-function isDifferent(item1, item2) {
-  try {
-    if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) {
-      propMatch(item1.parent, item2.parent, 'region_a');
-      propMatch(item1.parent, item2.parent, 'country');
-      propMatch(item1.parent, item2.parent, 'locality');
-      propMatch(item1.parent, item2.parent, 'neighbourhood');
-    }
-    else if (item1.parent !== item2.parent) {
-      throw new Error('different');
-    }
-
-    if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
-      for (var lang in item1.name) {
-        if(item2.name[lang] || lang === 'default') {
-          // do not consider absence of an additional name as a difference
-          propMatch(item1.name, item2.name, lang);
-        }
-      }
-    }
-    else {
-      propMatch(item1, item2, 'name');
-    }
-
-    if (item1.hasOwnProperty('address_parts') && item2.hasOwnProperty('address_parts')) {
-      propMatch(item1.address_parts, item2.address_parts, 'number');
-      propMatch(item1.address_parts, item2.address_parts, 'street');
-      propMatch(item1.address_parts, item2.address_parts, 'zip');
-    }
-    else if (item1.address_parts !== item2.address_parts) {
-      throw new Error('different');
-    }
-  }
-  catch (err) {
-    if (err.message === 'different') {
-      return true;
-    }
-    throw err;
-  }
-
-  return false;
-}
-
-/**
- * Throw exception if properties are different
- *
- * @param {object} item1
- * @param {object} item2
- * @param {string} prop
- * @throws {Error}
- */
-function propMatch(item1, item2, prop) {
-  var prop1 = item1[prop];
-  var prop2 = item2[prop];
-
-  // in the case the property is an array (currently only in parent schema)
-  // simply take the 1st item. this will change in the near future to support multiple hierarchies
-  if (_.isArray(prop1)) { prop1 = prop1[0]; }
-  if (_.isArray(prop2)) { prop2 = prop2[0]; }
-
-  if (normalizeString(prop1) !== normalizeString(prop2)) {
-    throw new Error('different');
-  }
-}
-
-/**
- * Remove punctuation and lowercase
- *
- * @param {string} str
- * @returns {string}
- */
-function normalizeString(str) {
-  if (!str) {
-    return '';
-  }
-  return str.toLowerCase().split(/[ ,-]+/).join(' ');
-}
-
-
 module.exports = setup;
diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js
index e4780097..8b61535f 100644
--- a/test/unit/fixture/dedupe_elasticsearch_results.js
+++ b/test/unit/fixture/dedupe_elasticsearch_results.js
@@ -1,377 +1,1192 @@
 module.exports = [
   {
     'center_point': {
-      'lon': -76.207456,
-      'lat': 40.039265
+      'lon': -76.293127,
+      'lat': 40.032787
     },
-    'address_parts': {},
     'parent': {
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA'],
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter High School'
+      'default': 'Hand Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'node:357289197',
     'category': [
       'education'
     ],
-    '_id': '357321757',
+    'layer': 'venue',
+    '_id': 'node:357289197',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
-  { // same as above, but change the neighbourhood
+  {
     'center_point': {
-      'lon': -77.207456,
-      'lat': 41.039265
+      'lon': -76.32746,
+      'lat': 40.02343
     },
-    'address': {},
     'parent': {
-      'localadmin': 'East Lampeter',
-      'region_a': 'PA',
-      'region': 'Pennsylvania',
-      'locality': 'Smoketown',
-      'country_a': 'USA',
-      'county': 'Lancaster County',
-      'country': 'United States',
-      'neighbourhood': 'Blueland' // ###
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487185'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [],
+      'locality_id': [],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter High School'
+      'default': 'Wheatland Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5219083',
     'category': [
       'education'
     ],
-    '_id': '357321757',
+    'layer': 'venue',
+    '_id': '5219083',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
-  { // same as #1, but change the locality
+  {
     'center_point': {
-      'lon': -73.207456,
-      'lat': 42.039265
+      'lon': -76.30107,
+      'lat': 40.05926
     },
-    'address': {},
     'parent': {
-      'localadmin': 'East Lampeter',
-      'region_a': 'PA',
-      'region': 'Pennsylvania',
-      'locality': 'Firetown', // ###
-      'country_a': 'USA',
-      'county': 'Lancaster County',
-      'country': 'United States',
-      'neighbourhood': 'Greenland'
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Rossmere'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85846173'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter High School'
+      'default': 'Catholic High School Stadium'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5183465',
     'category': [
-      'education'
+      'entertainment',
+      'recreation'
     ],
-    '_id': '357321757',
+    'layer': 'venue',
+    '_id': '5183465',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
-  { // same as #1, but with an additional name
+  {
     'center_point': {
-      'lon': -76.207456,
-      'lat': 40.039265
+      'lon': -76.285474,
+      'lat': 40.048535
     },
-    'address_parts': {},
     'parent': {
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA'],
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Grandview Heights'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85822505'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter High School',
-      'alt': 'High School of East Lampeter',
+      'default': 'McCaskey East High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'node:368338500',
     'category': [
       'education'
     ],
-    '_id': '357321757',
+    'layer': 'venue',
+    '_id': 'node:368338500',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.207456,
-      'lat': 40.039265
+      'lon': -76.327063,
+      'lat': 40.031869
     },
-    'address_parts': {},
     'parent': {
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA'],
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487185'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [],
+      'locality_id': [],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
+    'bounding_box': '{\'min_lat\':40.031513,\'max_lat\':40.032233,\'min_lon\':-76.328429,\'max_lon\':-76.326216}',
     'name': {
-      'default': 'East Lampeter, High-School'
+      'default': 'Wheatland Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'way:84969670',
     'category': [
       'education'
     ],
-    '_id': '357321757',
+    'layer': 'venue',
+    '_id': 'way:84969670',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.23246,
-      'lat': 39.99288
+      'lon': -76.29274,
+      'lat': 40.03288
     },
-    'address_parts': {},
     'parent': {
-      'localadmin': ['West Lampeter'],
-      'region_a': ['PA'],
-      'region': ['Pennsylvania'],
-      'locality': ['Lampeter'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Wheatland Mills']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'Lampeter-Strasburg High School'
+      'default': 'Hand Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5192545',
     'category': [
       'education'
     ],
-    '_id': '4559068',
-    '_type': 'geoname',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    'layer': 'venue',
+    '_id': '5192545',
+    '_type': 'venue',
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.20746,
-      'lat': 40.03927
+      'lon': -76.28496,
+      'lat': 40.04732
     },
-    'address_parts': {},
     'parent': {
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA'],
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Grandview Heights'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85822505'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter High School'
+      'default': 'Lincoln Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5198085',
     'category': [
       'education'
     ],
-    '_id': '5187980',
-    '_type': 'geoname',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    'layer': 'venue',
+    '_id': '5198085',
+    '_type': 'venue',
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.232457,
-      'lat': 39.992877
+      'lon': -76.31857,
+      'lat': 40.04204
     },
-    'address_parts': {},
     'parent': {
-      'region': ['Pennsylvania'],
-      'locality': ['Lampeter'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Wheatland Mills'],
-      'localadmin': ['West Lampeter'],
-      'region_a': ['PA']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'Lampeter-Strasburg High School'
+      'default': 'Reynolds Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5208101',
     'category': [
       'education'
     ],
-    '_id': '357294404',
+    'layer': 'venue',
+    '_id': '5208101',
     '_type': 'venue',
-    '_score': 1.2367082,
-    'confidence': 0.879
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.207456,
-      'lat': 40.038987
+      'lon': -76.290392,
+      'lat': 40.048281
     },
-    'address_parts': {},
     'parent': {
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland'],
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Grandview Heights'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85822505'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
+    'bounding_box': '{\'min_lat\':40.047288,\'max_lat\':40.049171,\'min_lon\':-76.291609,\'max_lon\':-76.289314}',
     'name': {
-      'default': 'East Lampeter School'
+      'default': 'McCaskey High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'way:161088588',
     'category': [
       'education'
     ],
-    '_id': '357283977',
+    'layer': 'venue',
+    '_id': 'way:161088588',
     '_type': 'venue',
-    '_score': 1.1036991,
-    'confidence': 0.664
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -76.20746,
-      'lat': 40.03899
+      'lon': -76.29051,
+      'lat': 40.04788
     },
-    'address_parts': {},
     'parent': {
-      'region': ['Pennsylvania'],
-      'locality': ['Smoketown'],
-      'country_a': ['USA'],
-      'county': ['Lancaster County'],
-      'country': ['United States'],
-      'neighbourhood': ['Greenland'],
-      'localadmin': ['East Lampeter'],
-      'region_a': ['PA']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Grandview Heights'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85822505'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
     'name': {
-      'default': 'East Lampeter School'
+      'default': 'McCaskey High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5200263',
     'category': [
       'education'
     ],
-    '_id': '5187966',
-    '_type': 'geoname',
-    '_score': 1.1036991,
-    'confidence': 0.664
+    'layer': 'venue',
+    '_id': '5200263',
+    '_type': 'venue',
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -94.167445,
-      'lat': 38.762788
+      'lon': -76.318983,
+      'lat': 40.042051
     },
-    'address_parts': {},
     'parent': {
-      'region': ['Missouri'],
-      'locality': ['Strasburg'],
-      'country_a': ['USA'],
-      'county': ['Cass County'],
-      'country': ['United States'],
-      'localadmin': ['Polk'],
-      'region_a': ['MO']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
+    'bounding_box': '{\'min_lat\':40.041542,\'max_lat\':40.042777,\'min_lon\':-76.31963,\'max_lon\':-76.318094}',
     'name': {
-      'default': 'Strasburg School'
+      'default': 'Reynolds Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'way:34212977',
     'category': [
       'education'
     ],
-    '_id': '358058986',
+    'layer': 'venue',
+    '_id': 'way:34212977',
     '_type': 'venue',
-    '_score': 1.0492544,
-    'confidence': 0.658
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -78.36317,
-      'lat': 38.98445
-    },
-    'address_parts': {},
-    'name': {
-      'default': 'Strasburg High School'
+      'lon': -76.284958,
+      'lat': 40.04732
     },
     'parent': {
-      'region_a': ['VA'],
-      'region': ['Virginia'],
-      'locality': ['Strasburg'],
-      'country_a': ['USA'],
-      'county': ['Shenandoah County'],
-      'country': ['United States'],
-      'neighbourhood': ['Strasburg Junction']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [
+        'Grandview Heights'
+      ],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [
+        '85822505'
+      ],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [
+        null
+      ],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
+    },
+    'name': {
+      'default': 'Lincoln Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'node:357330916',
     'category': [
       'education'
     ],
-    '_id': '4787978',
-    '_type': 'geoname',
-    '_score': 0.9724125,
-    'confidence': 0.649
+    'layer': 'venue',
+    '_id': 'node:357330916',
+    '_type': 'venue',
+    '_score': 0.47265986,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -100.16516,
-      'lat': 46.13427
-    },
-    'address_parts': {},
-    'name': {
-      'default': 'Strasburg High School'
+      'lon': -76.280791,
+      'lat': 40.045098
     },
     'parent': {
-      'localadmin': ['Strasburg'],
-      'region_a': ['ND'],
-      'region': ['North Dakota'],
-      'locality': ['Strasburg'],
-      'country_a': ['USA'],
-      'county': ['Emmons County'],
-      'country': ['United States']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
+    },
+    'name': {
+      'default': 'Lancaster Christian Junior High School'
     },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'openstreetmap',
+    'source_id': 'node:357330919',
     'category': [
       'education'
     ],
-    '_id': '9683163',
-    '_type': 'geoname',
-    '_score': 0.9724125,
-    'confidence': 0.649
+    'layer': 'venue',
+    '_id': 'node:357330919',
+    '_type': 'venue',
+    '_score': 0.4432487,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   },
   {
     'center_point': {
-      'lon': -81.532392,
-      'lat': 40.597578
-    },
-    'address_parts': {},
-    'name': {
-      'default': 'Strasburg High School'
+      'lon': -76.28079,
+      'lat': 40.0451
     },
     'parent': {
-      'localadmin': ['Franklin'],
-      'region_a': ['OH'],
-      'region': ['Ohio'],
-      'locality': ['Strasburg'],
-      'country_a': ['USA'],
-      'county': ['Tuscarawas County'],
-      'country': ['United States']
+      'country': [
+        'United States'
+      ],
+      'macrocounty_a': [],
+      'locality_a': [
+        null
+      ],
+      'county': [
+        'Lancaster County'
+      ],
+      'borough_a': [],
+      'borough_id': [],
+      'borough': [],
+      'macroregion': [],
+      'region_a': [
+        'PA'
+      ],
+      'localadmin': [
+        'Lancaster'
+      ],
+      'macrocounty': [],
+      'county_id': [
+        '102081377'
+      ],
+      'neighbourhood': [],
+      'localadmin_id': [
+        '404487183'
+      ],
+      'macroregion_id': [],
+      'neighbourhood_id': [],
+      'country_a': [
+        'USA'
+      ],
+      'macroregion_a': [],
+      'localadmin_a': [
+        null
+      ],
+      'region_id': [
+        '85688481'
+      ],
+      'locality': [
+        'Lancaster'
+      ],
+      'locality_id': [
+        '101718643'
+      ],
+      'neighbourhood_a': [],
+      'region': [
+        'Pennsylvania'
+      ],
+      'macrocounty_id': [],
+      'country_id': [
+        '85633793'
+      ],
+      'county_a': [
+        null
+      ]
     },
+    'name': {
+      'default': 'Lancaster Christian Junior High School'
+    },
+    'address_parts': {},
+    'alpha3': 'USA',
+    'source': 'geonames',
+    'source_id': '5197082',
     'category': [
       'education'
     ],
-    '_id': '356646971',
+    'layer': 'venue',
+    '_id': '5197082',
     '_type': 'venue',
-    '_score': 0.9724125,
-    'confidence': 0.649
+    '_score': 0.4432487,
+    '_matched_queries': [
+      'fallback.venue'
+    ],
+    'confidence': 0.5
   }
-];
+];
\ No newline at end of file
diff --git a/test/unit/helper/diffPlaces.js b/test/unit/helper/diffPlaces.js
new file mode 100644
index 00000000..a7dd692d
--- /dev/null
+++ b/test/unit/helper/diffPlaces.js
@@ -0,0 +1,180 @@
+var isDifferent= require('../../../helper/diffPlaces').isDifferent;
+
+module.exports.tests = {};
+
+module.exports.tests.dedupe = function(test, common) {
+
+  test('match same object', function(t) {
+    var item1 = {
+      'parent': {
+        'country': [ 'United States' ],
+        'county': [ 'Otsego County' ],
+        'region_a': [ 'NY' ],
+        'localadmin': [ 'Cherry Valley' ],
+        'county_id': [ '102082399' ],
+        'localadmin_id': [ '404522887' ],
+        'country_a': [ 'USA' ],
+        'region_id': [ '85688543' ],
+        'locality': [ 'Cherry Valley' ],
+        'locality_id': [ '85978799' ],
+        'region': [ 'New York' ],
+        'country_id': [ '85633793' ]
+      },
+      'name': {
+        'default': '1 Main Street'
+      },
+      'address_parts': {
+        'number': '1',
+        'street': 'Main Street'
+      },
+      'layer': 'address'
+    };
+
+    t.false(isDifferent(item1, item1), 'should be the same');
+    t.end();
+  });
+
+  test('catch diff layers', function(t) {
+    var item1 = { 'layer': 'address' };
+    var item2 = { 'layer': 'venue' };
+
+    t.true(isDifferent(item1, item2), 'should be different');
+    t.end();
+  });
+
+  test('catch diff parent', function(t) {
+    var item1 = {
+      'layer': 'same',
+      'parent': {
+        'country_id': '12345'
+      }
+    };
+    var item2 = {
+      'layer': 'same',
+      'parent': {
+        'country_id': '54321'
+      }
+    };
+
+    t.true(isDifferent(item1, item2), 'should be different');
+    t.end();
+  });
+
+  test('catch diff name', function(t) {
+    var item1 = {
+      'name': {
+        'default': '1 Main St'
+      }
+    };
+    var item2 = {
+      'name': {
+        'default': '1 Broad St'
+      }
+    };
+
+    t.true(isDifferent(item1, item2), 'should be different');
+    t.end();
+  });
+
+  test('match diff capitalization in name', function(t) {
+    var item1 = {
+      'name': {
+        'default': '1 MAIN ST'
+      }
+    };
+    var item2 = {
+      'name': {
+        'default': '1 Main St'
+      }
+    };
+
+    t.false(isDifferent(item1, item2), 'should be the same');
+    t.end();
+  });
+
+  test('do not handle expansions', function(t) {
+    // we currently don't handle expansions and abbreviations and
+    // this is a test waiting to be updated as soon as we fix it
+
+    var item1 = {
+      'name': {
+        'default': '1 Main Street'
+      }
+    };
+    var item2 = {
+      'name': {
+        'default': '1 Main St'
+      }
+    };
+
+    t.true(isDifferent(item1, item2), 'should be different');
+    t.end();
+  });
+
+  test('missing names in other langs should not be a diff', function(t) {
+    var item1 = {
+      'name': {
+        'default': 'Moscow',
+        'rus': 'Москва'
+      }
+    };
+    var item2 = {
+      'name': {
+        'default': 'Moscow'
+      }
+    };
+
+    t.false(isDifferent(item1, item2), 'should be the same');
+    t.end();
+  });
+
+  test('catch diff address', function(t) {
+    var item1 = {
+      'address_parts': {
+        'number': '1',
+        'street': 'Main Street',
+        'zip': '90210'
+      }
+    };
+    var item2 = {
+      'address_parts': {
+        'number': '2',
+        'street': 'Main Street',
+        'zip': '90210'
+      }
+    };
+
+    t.true(isDifferent(item1, item2), 'should be different');
+    t.end();
+  });
+
+  test('catch diff address', function(t) {
+    var item1 = {
+      'address_parts': {
+        'number': '1',
+        'street': 'Main Street',
+        'zip': '90210'
+      }
+    };
+    var item2 = {
+      'address_parts': {
+        'number': '1',
+        'street': 'Main Street'
+      }
+    };
+
+    t.false(isDifferent(item1, item2), 'should be the same');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[helper] diffPlaces: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/test/unit/middleware/dedupe.js b/test/unit/middleware/dedupe.js
index ad553f9c..b8100955 100644
--- a/test/unit/middleware/dedupe.js
+++ b/test/unit/middleware/dedupe.js
@@ -16,7 +16,7 @@ module.exports.tests.dedupe = function(test, common) {
       data: data
     };
 
-    var expectedCount = 9;
+    var expectedCount = 8;
     dedupe(req, res, function () {
       t.equal(res.data.length, expectedCount, 'results have fewer items than before');
       t.end();
diff --git a/test/unit/run.js b/test/unit/run.js
index 77fb3415..a8b4b252 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -12,6 +12,7 @@ var tests = [
   require('./controller/index'),
   require('./controller/place'),
   require('./controller/search'),
+  require('./helper/diffPlaces'),
   require('./helper/geojsonify'),
   require('./helper/labelGenerator_examples'),
   require('./helper/labelGenerator_default'),

From 6049772a5d5dba3ab097562819686d6398322bb0 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Tue, 6 Sep 2016 08:59:49 -0400
Subject: [PATCH 46/82] Fix require path

---
 helper/diffPlaces.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helper/diffPlaces.js b/helper/diffPlaces.js
index e6a48af5..90cfa017 100644
--- a/helper/diffPlaces.js
+++ b/helper/diffPlaces.js
@@ -1,5 +1,5 @@
 var _ = require('lodash');
-var placeTypes = require('../helper/placeTypes');
+var placeTypes = require('./placeTypes');
 
 /**
  * Compare the layer properties if they exist.

From efa0818ade73de060b57e99405a869619e4083e5 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Tue, 6 Sep 2016 09:02:28 -0400
Subject: [PATCH 47/82] Rename isDiff* functions to be more intuitive

---
 helper/diffPlaces.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/helper/diffPlaces.js b/helper/diffPlaces.js
index 90cfa017..c13e5827 100644
--- a/helper/diffPlaces.js
+++ b/helper/diffPlaces.js
@@ -11,7 +11,7 @@ var placeTypes = require('./placeTypes');
  * @returns {boolean}
  * @throws {Error}
  */
-function isDiffLayer(item1, item2) {
+function assertLayerMatch(item1, item2) {
   if (item1.layer === item2.layer) {
     return false;
   }
@@ -29,7 +29,7 @@ function isDiffLayer(item1, item2) {
  * @returns {boolean}
  * @throws {Error}
  */
-function isDiffParentHierarchy(item1, item2) {
+function assertParentHierarchyMatch(item1, item2) {
   // if neither object has parent, assume same
   if (!item1.hasOwnProperty('parent') && !item2.hasOwnProperty('parent')) {
     return false;
@@ -61,7 +61,7 @@ function isDiffParentHierarchy(item1, item2) {
  * @returns {boolean}
  * @throws {Error}
  */
-function isDiffName(item1, item2) {
+function assertNameMatch(item1, item2) {
   if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
     for (var lang in item1.name) {
       if(item2.name[lang] || lang === 'default') {
@@ -85,7 +85,7 @@ function isDiffName(item1, item2) {
  * @returns {boolean}
  * @throws {Error}
  */
-function isDiffAddress(item1, item2) {
+function assertAddressMatch(item1, item2) {
   // if neither record has address, assume same
   if (!item1.hasOwnProperty('address_parts') && !item2.hasOwnProperty('address_parts')) {
     return false;
@@ -119,10 +119,10 @@ function isDiffAddress(item1, item2) {
  */
 function isDifferent(item1, item2) {
   try {
-    isDiffLayer(item1, item2);
-    isDiffParentHierarchy(item1, item2);
-    isDiffName(item1, item2);
-    isDiffAddress(item1, item2);
+    assertLayerMatch(item1, item2);
+    assertParentHierarchyMatch(item1, item2);
+    assertNameMatch(item1, item2);
+    assertAddressMatch(item1, item2);
   }
   catch (err) {
     if (err.message === 'different') {

From d8e0b0135809b3bb4f501326d357180192b29957 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Tue, 6 Sep 2016 09:10:32 -0400
Subject: [PATCH 48/82] Cleanup a few small things

---
 helper/diffPlaces.js | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/helper/diffPlaces.js b/helper/diffPlaces.js
index c13e5827..712c7959 100644
--- a/helper/diffPlaces.js
+++ b/helper/diffPlaces.js
@@ -20,7 +20,7 @@ function assertLayerMatch(item1, item2) {
 }
 
 /**
- * Compare the parent.* properties if they exist.
+ * Compare the parent.*_id properties if they exist.
  * Returns false if the objects are the same, and throws
  * an exception with the message 'different' if not.
  *
@@ -64,7 +64,7 @@ function assertParentHierarchyMatch(item1, item2) {
 function assertNameMatch(item1, item2) {
   if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
     for (var lang in item1.name) {
-      if(item2.name[lang] || lang === 'default') {
+      if(item2.name.hasOwnProperty(lang) || lang === 'default') {
         // do not consider absence of an additional name as a difference
         propMatch(item1.name, item2.name, lang);
       }
@@ -163,9 +163,14 @@ function propMatch(item1, item2, prop) {
  * @returns {string}
  */
 function normalizeString(str) {
-  if (!str) {
+  if (!_.isString(str)) {
+    return str;
+  }
+
+  if (_.isEmpty(str)) {
     return '';
   }
+
   return str.toLowerCase().split(/[ ,-]+/).join(' ');
 }
 

From e33c58bd42874bb004e929698cbe3ba470952490 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Tue, 6 Sep 2016 09:27:25 -0400
Subject: [PATCH 49/82] Remove empty arrays from ES fixture

---
 .../fixture/dedupe_elasticsearch_results.js   | 156 ------------------
 1 file changed, 156 deletions(-)

diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js
index 8b61535f..39c8e71f 100644
--- a/test/unit/fixture/dedupe_elasticsearch_results.js
+++ b/test/unit/fixture/dedupe_elasticsearch_results.js
@@ -8,37 +8,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -51,11 +41,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -91,48 +79,33 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
-      'locality_a': [],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487185'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
       'region_id': [
         '85688481'
       ],
-      'locality': [],
-      'locality_id': [],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -168,24 +141,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -195,14 +162,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85846173'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -221,7 +186,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -258,24 +222,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -285,14 +243,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85822505'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -311,7 +267,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -347,48 +302,33 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
-      'locality_a': [],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487185'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
       'region_id': [
         '85688481'
       ],
-      'locality': [],
-      'locality_id': [],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -425,37 +365,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -468,11 +398,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -508,24 +436,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -535,14 +457,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85822505'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -561,7 +481,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -597,37 +516,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -640,11 +549,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -680,24 +587,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -707,14 +608,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85822505'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -733,7 +632,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -770,24 +668,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -797,14 +689,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85822505'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -823,7 +713,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -859,37 +748,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -902,11 +781,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -943,24 +820,18 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
@@ -970,14 +841,12 @@ module.exports = [
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
       'neighbourhood_id': [
         '85822505'
       ],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -996,7 +865,6 @@ module.exports = [
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -1032,37 +900,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -1075,11 +933,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],
@@ -1115,37 +971,27 @@ module.exports = [
       'country': [
         'United States'
       ],
-      'macrocounty_a': [],
       'locality_a': [
         null
       ],
       'county': [
         'Lancaster County'
       ],
-      'borough_a': [],
-      'borough_id': [],
-      'borough': [],
-      'macroregion': [],
       'region_a': [
         'PA'
       ],
       'localadmin': [
         'Lancaster'
       ],
-      'macrocounty': [],
       'county_id': [
         '102081377'
       ],
-      'neighbourhood': [],
       'localadmin_id': [
         '404487183'
       ],
-      'macroregion_id': [],
-      'neighbourhood_id': [],
       'country_a': [
         'USA'
       ],
-      'macroregion_a': [],
       'localadmin_a': [
         null
       ],
@@ -1158,11 +1004,9 @@ module.exports = [
       'locality_id': [
         '101718643'
       ],
-      'neighbourhood_a': [],
       'region': [
         'Pennsylvania'
       ],
-      'macrocounty_id': [],
       'country_id': [
         '85633793'
       ],

From 9f350eb117973cc4f603c0f3b5d42c39140e428c Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 7 Sep 2016 08:33:39 -0400
Subject: [PATCH 50/82] updated tests to reflect changes to query module

---
 test/unit/fixture/search_boundary_country.js  |  15 +-
 test/unit/fixture/search_fallback.js          | 204 +++++++++++++++++-
 test/unit/fixture/search_geodisambiguation.js |  15 +-
 test/unit/fixture/search_linguistic_bbox.js   |  15 +-
 test/unit/fixture/search_linguistic_focus.js  |  15 +-
 .../fixture/search_linguistic_focus_bbox.js   |  15 +-
 .../search_linguistic_focus_null_island.js    |  15 +-
 test/unit/fixture/search_linguistic_only.js   |  15 +-
 .../fixture/search_linguistic_viewport.js     |  15 +-
 ...search_linguistic_viewport_min_diagonal.js |  15 +-
 .../fixture/search_with_category_filtering.js |  15 +-
 .../fixture/search_with_source_filtering.js   |  15 +-
 12 files changed, 349 insertions(+), 20 deletions(-)

diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js
index 977b15c9..ea653da8 100644
--- a/test/unit/fixture/search_boundary_country.js
+++ b/test/unit/fixture/search_boundary_country.js
@@ -55,5 +55,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
index 6c076c61..12b617a3 100644
--- a/test/unit/fixture/search_fallback.js
+++ b/test/unit/fixture/search_fallback.js
@@ -69,7 +69,9 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -107,6 +109,11 @@ module.exports = {
                           'address_parts.street': 'street value'
                         }
                       },
+                      {
+                        'match_phrase': {
+                          'address_parts.zip': 'postalcode value'
+                        }
+                      },
                       {
                         'multi_match': {
                           'query': 'neighbourhood value',
@@ -157,7 +164,9 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -235,7 +244,9 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -303,7 +314,9 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -359,7 +372,9 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -385,8 +400,18 @@ module.exports = {
                 },
                 {
                   'bool': {
-                    '_name': 'fallback.county',
+                    '_name': 'fallback.localadmin',
                     'must': [
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
                       {
                         'multi_match': {
                           'query': 'county value',
@@ -405,7 +430,55 @@ module.exports = {
                           'type': 'phrase',
                           'fields': [
                             'parent.region',
-                            'parent.region_a'
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'localadmin'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.county',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
                           ]
                         }
                       },
@@ -429,6 +502,52 @@ module.exports = {
                     }
                   }
                 },
+                {
+                  'bool': {
+                    '_name': 'fallback.macrocounty',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'macrocounty'
+                      }
+                    }
+                  }
+                },
                 {
                   'bool': {
                     '_name': 'fallback.region',
@@ -465,8 +584,18 @@ module.exports = {
                 },
                 {
                   'bool': {
-                    '_name': 'fallback.country',
+                    '_name': 'fallback.macroregion',
                     'must': [
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      },
                       {
                         'multi_match': {
                           'query': 'country value',
@@ -480,6 +609,50 @@ module.exports = {
                         }
                       }
                     ],
+                    'filter': {
+                      'term': {
+                        'layer': 'macroregion'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.dependency',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'dependency'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.country',
+                    'must': [
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a'
+                          ]
+                        }
+                      }
+                    ],
                     'filter': {
                       'term': {
                         'layer': 'country'
@@ -521,5 +694,18 @@ module.exports = {
     }
   },
   'size': 20,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_geodisambiguation.js b/test/unit/fixture/search_geodisambiguation.js
index 34b447e1..5db0af98 100644
--- a/test/unit/fixture/search_geodisambiguation.js
+++ b/test/unit/fixture/search_geodisambiguation.js
@@ -250,5 +250,18 @@ module.exports = {
     }
   },
   'size': 20,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js
index 9470529d..e6582e29 100644
--- a/test/unit/fixture/search_linguistic_bbox.js
+++ b/test/unit/fixture/search_linguistic_bbox.js
@@ -58,5 +58,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js
index 25cefa2e..6af7efcf 100644
--- a/test/unit/fixture/search_linguistic_focus.js
+++ b/test/unit/fixture/search_linguistic_focus.js
@@ -61,5 +61,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js
index 6523232f..4126d479 100644
--- a/test/unit/fixture/search_linguistic_focus_bbox.js
+++ b/test/unit/fixture/search_linguistic_focus_bbox.js
@@ -72,5 +72,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js
index c6ccdd40..9e7b299c 100644
--- a/test/unit/fixture/search_linguistic_focus_null_island.js
+++ b/test/unit/fixture/search_linguistic_focus_null_island.js
@@ -61,5 +61,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js
index 08254737..37767375 100644
--- a/test/unit/fixture/search_linguistic_only.js
+++ b/test/unit/fixture/search_linguistic_only.js
@@ -47,5 +47,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js
index 08254737..37767375 100644
--- a/test/unit/fixture/search_linguistic_viewport.js
+++ b/test/unit/fixture/search_linguistic_viewport.js
@@ -47,5 +47,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
index 08254737..37767375 100644
--- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
+++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
@@ -47,5 +47,18 @@ module.exports = {
     }
   },
   'size': 10,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_with_category_filtering.js b/test/unit/fixture/search_with_category_filtering.js
index 3c217c35..9aa12a66 100644
--- a/test/unit/fixture/search_with_category_filtering.js
+++ b/test/unit/fixture/search_with_category_filtering.js
@@ -48,5 +48,18 @@ module.exports = {
     }
   },
   'size': 20,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };
diff --git a/test/unit/fixture/search_with_source_filtering.js b/test/unit/fixture/search_with_source_filtering.js
index 454c317d..0fdbdc24 100644
--- a/test/unit/fixture/search_with_source_filtering.js
+++ b/test/unit/fixture/search_with_source_filtering.js
@@ -47,5 +47,18 @@ module.exports = {
     }
   },
   'size': 20,
-  'track_scores': true
+  'track_scores': true,
+  'sort': [
+    {
+      'population': {
+        'order': 'desc'
+      }
+    },
+    {
+      'popularity': {
+        'order': 'desc'
+      }
+    },
+    '_score'
+  ]
 };

From 681d689f51736ea7e44245fcc8b039a960cb1c85 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 7 Sep 2016 08:45:41 -0400
Subject: [PATCH 51/82] fixed test names

---
 test/unit/middleware/trimByGranularity.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/unit/middleware/trimByGranularity.js b/test/unit/middleware/trimByGranularity.js
index 07e6db9a..f94ad608 100644
--- a/test/unit/middleware/trimByGranularity.js
+++ b/test/unit/middleware/trimByGranularity.js
@@ -134,7 +134,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
-  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
+  test('all records with fallback.* matched_queries name should retain only counties when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
@@ -162,7 +162,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
-  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
+  test('all records with fallback.* matched_queries name should retain only regions when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {

From 5178dc1cc92cfcc72c3d86932cda37d7bdbadb3c Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 7 Sep 2016 08:47:43 -0400
Subject: [PATCH 52/82] added `trimByGranularity` to tests-to-run

---
 test/unit/run.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/unit/run.js b/test/unit/run.js
index a8b4b252..1cd8ab45 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -33,6 +33,7 @@ var tests = [
   require('./middleware/parseBBox'),
   require('./middleware/sendJSON'),
   require('./middleware/normalizeParentIds'),
+  require('./middleware/trimByGranularity'),
   require('./query/autocomplete'),
   require('./query/autocomplete_defaults'),
   require('./query/search_defaults'),

From d5e993fc11a5ec331bed55a779f993ad16753c4d Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 7 Sep 2016 08:58:51 -0400
Subject: [PATCH 53/82] included `localadmin`, `macrocounty`, `macroregion`,
 and `dependency`

---
 middleware/trimByGranularity.js           |   4 +
 test/unit/middleware/trimByGranularity.js | 138 ++++++++++++++++++++++
 2 files changed, 142 insertions(+)

diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js
index 1fade0cc..d5ff528b 100644
--- a/middleware/trimByGranularity.js
+++ b/middleware/trimByGranularity.js
@@ -19,8 +19,12 @@ var layers = [
   'neighbourhood',
   'borough',
   'locality',
+  'localadmin',
   'county',
+  'macrocounty',
   'region',
+  'macroregion',
+  'dependency',
   'country'
 ];
 
diff --git a/test/unit/middleware/trimByGranularity.js b/test/unit/middleware/trimByGranularity.js
index f94ad608..fdb1e839 100644
--- a/test/unit/middleware/trimByGranularity.js
+++ b/test/unit/middleware/trimByGranularity.js
@@ -22,8 +22,12 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'address 1', _matched_queries: ['fallback.address'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
         { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
         { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -53,8 +57,12 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'address 2', _matched_queries: ['fallback.address'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
         { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
         { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -83,8 +91,12 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
         { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
         { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -112,8 +124,12 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
         { name: 'locality 2', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
         { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
         { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -134,6 +150,38 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
+  test('all records with fallback.* matched_queries name should retain only localadmins when they are most granular', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
+        { name: 'localadmin 2', _matched_queries: ['fallback.localadmin'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
+      { name: 'localadmin 2', _matched_queries: ['fallback.localadmin'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only localadmin records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
+
   test('all records with fallback.* matched_queries name should retain only counties when they are most granular', function(t) {
     var req = { clean: {} };
 
@@ -141,7 +189,10 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'county 1', _matched_queries: ['fallback.county'] },
         { name: 'county 2', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
         { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -162,6 +213,36 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
+  test('all records with fallback.* matched_queries name should retain only macrocounties when they are most granular', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
+        { name: 'macrocounty 2', _matched_queries: ['fallback.macrocounty'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
+      { name: 'macrocounty 2', _matched_queries: ['fallback.macrocounty'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only macrocounty records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
+
   test('all records with fallback.* matched_queries name should retain only regions when they are most granular', function(t) {
     var req = { clean: {} };
 
@@ -169,6 +250,8 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'region 1', _matched_queries: ['fallback.region'] },
         { name: 'region 2', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
         { name: 'country 1', _matched_queries: ['fallback.country'] },
         { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
@@ -189,6 +272,61 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
+  test('all records with fallback.* matched_queries name should retain only macroregions when they are most granular', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'macroregion 2', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+      { name: 'macroregion 2', _matched_queries: ['fallback.macroregion'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only macroregion records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
+
+  test('all records with fallback.* matched_queries name should retain only dependencies when they are most granular', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'dependency 2', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+      { name: 'dependency 2', _matched_queries: ['fallback.dependency'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only dependency records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
+
   test('all records with fallback.* matched_queries name should retain only countries when they are most granular', function(t) {
     var req = { clean: {} };
 

From 6f4ec874fe6a290d10977e3fa48263c0eb3d1376 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 7 Sep 2016 15:45:01 -0400
Subject: [PATCH 54/82] added special override condition for libpostal
 query/housenumber mixup

---
 query/text_parser.js           |  29 +++++++
 test/unit/query/text_parser.js | 153 ++++++++++++++++++++++++++-------
 test/unit/run.js               |   1 +
 3 files changed, 150 insertions(+), 33 deletions(-)

diff --git a/query/text_parser.js b/query/text_parser.js
index c71e444a..293c021e 100644
--- a/query/text_parser.js
+++ b/query/text_parser.js
@@ -1,4 +1,5 @@
 var logger = require('pelias-logger').get('api');
+var _ = require('lodash');
 
 // all the address parsing logic
 function addParsedVariablesToQueryVariables( parsed_text, vs ){
@@ -61,6 +62,34 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
     vs.var( 'input:country', parsed_text.country );
   }
 
+  // libpostal sometimes parses addresses with prefix house numbers in places where
+  // the house number is normally postfix incorrectly, for instance:
+  // ```> 1 Grolmanstraße, Berlin, Germany
+  //
+  // Result:
+  //
+  // {
+  //   "house": "1",
+  //   "road": "grolmanstrasse",
+  //   "state": "berlin",
+  //   "country": "germany"
+  // }```
+  //
+  // In libpostal parlance, `house` is just a query term, not the house number.
+  // This special case moves the query term to the house number field if there's a street,
+  // there's no house number, and the query is parseable as an integer, then use the
+  // query as the house number and blank out the query.  
+  if (shouldSetQueryIntoHouseNumber(vs)) {
+    vs.var( 'input:housenumber', vs.var('input:query').toString());
+    vs.unset( 'input:query' );
+  }
+
+}
+
+function shouldSetQueryIntoHouseNumber(vs) {
+  return !vs.isset('input:housenumber') &&
+          vs.isset('input:street') &&
+          /^[0-9]+$/.test(vs.var('input:query').toString());
 }
 
 module.exports = addParsedVariablesToQueryVariables;
diff --git a/test/unit/query/text_parser.js b/test/unit/query/text_parser.js
index 5ffc5b76..274dc10c 100644
--- a/test/unit/query/text_parser.js
+++ b/test/unit/query/text_parser.js
@@ -10,57 +10,144 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-module.exports.tests.query = function(test, common) {
-  test('parsed_text without properties should leave vs properties unset', function(t) {
-    var parsed_text = {};
+// module.exports.tests.query = function(test, common) {
+//   test('parsed_text without properties should leave vs properties unset', function(t) {
+//     var parsed_text = {};
+//     var vs = new VariableStore();
+//
+//     text_parser(parsed_text, vs);
+//
+//     t.false(vs.isset('input:query'));
+//     t.false(vs.isset('input:category'));
+//     t.false(vs.isset('input:housenumber'));
+//     t.false(vs.isset('input:street'));
+//     t.false(vs.isset('input:neighbourhood'));
+//     t.false(vs.isset('input:borough'));
+//     t.false(vs.isset('input:postcode'));
+//     t.false(vs.isset('input:locality'));
+//     t.false(vs.isset('input:county'));
+//     t.false(vs.isset('input:region'));
+//     t.false(vs.isset('input:country'));
+//     t.end();
+//
+//   });
+//
+//   test('parsed_text without properties should leave vs properties unset', function(t) {
+//     var parsed_text = {
+//       query: 'query value',
+//       category: 'category value',
+//       number: 'number value',
+//       street: 'street value',
+//       neighbourhood: 'neighbourhood value',
+//       borough: 'borough value',
+//       postalcode: 'postalcode value',
+//       city: 'city value',
+//       county: 'county value',
+//       state: 'state value',
+//       country: 'country value'
+//     };
+//     var vs = new VariableStore();
+//
+//     text_parser(parsed_text, vs);
+//
+//     t.equals(vs.var('input:query').toString(), 'query value');
+//     t.equals(vs.var('input:category').toString(), 'category value');
+//     t.equals(vs.var('input:housenumber').toString(), 'number value');
+//     t.equals(vs.var('input:street').toString(), 'street value');
+//     t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value');
+//     t.equals(vs.var('input:borough').toString(), 'borough value');
+//     t.equals(vs.var('input:postcode').toString(), 'postalcode value');
+//     t.equals(vs.var('input:locality').toString(), 'city value');
+//     t.equals(vs.var('input:county').toString(), 'county value');
+//     t.equals(vs.var('input:region').toString(), 'state value');
+//     t.equals(vs.var('input:country').toString(), 'country value');
+//     t.end();
+//
+//   });
+//
+// };
+
+module.exports.tests.housenumber_special_cases = function(test, common) {
+  test('numeric query with street but no number should reassign query to housenumber', function(t) {
+    var parsed_text = {
+      query: '17',
+      // no house number set
+      street: 'street value'
+    };
     var vs = new VariableStore();
 
     text_parser(parsed_text, vs);
 
     t.false(vs.isset('input:query'));
-    t.false(vs.isset('input:category'));
+    t.equals(vs.var('input:housenumber').toString(), '17');
+    t.equals(vs.var('input:street').toString(), 'street value');
+    t.end();
+
+  });
+
+  test('numeric query with street but without number should not change anything', function(t) {
+    var parsed_text = {
+      query: '17',
+      number: 'housenumber value',
+      street: 'street value'
+      // no number or street
+    };
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.equals(vs.var('input:query').toString(), '17');
+    t.equals(vs.var('input:housenumber').toString(), 'housenumber value');
+    t.equals(vs.var('input:street').toString(), 'street value');
+    t.end();
+
+  });
+
+  test('numeric query with number but without street should not change anything', function(t) {
+    var parsed_text = {
+      query: '17',
+      number: 'number value'
+      // no number or street
+    };
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.equals(vs.var('input:query').toString(), '17');
+    t.equals(vs.var('input:housenumber').toString(), 'number value');
+    t.false(vs.isset('input:street'));
+    t.end();
+
+  });
+
+  test('numeric query without street or number should not change anything', function(t) {
+    var parsed_text = {
+      query: '17'
+      // no number or street
+    };
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.equals(vs.var('input:query').toString(), '17');
     t.false(vs.isset('input:housenumber'));
     t.false(vs.isset('input:street'));
-    t.false(vs.isset('input:neighbourhood'));
-    t.false(vs.isset('input:borough'));
-    t.false(vs.isset('input:postcode'));
-    t.false(vs.isset('input:locality'));
-    t.false(vs.isset('input:county'));
-    t.false(vs.isset('input:region'));
-    t.false(vs.isset('input:country'));
     t.end();
 
   });
 
-  test('parsed_text without properties should leave vs properties unset', function(t) {
+  test('non-numeric query with street but no number should not change anything', function(t) {
     var parsed_text = {
-      query: 'query value',
-      category: 'category value',
-      number: 'number value',
-      street: 'street value',
-      neighbourhood: 'neighbourhood value',
-      borough: 'borough value',
-      postalcode: 'postalcode value',
-      city: 'city value',
-      county: 'county value',
-      state: 'state value',
-      country: 'country value'
+      query: '13 this is 15 not a number 17',
+      street: 'street value'
     };
     var vs = new VariableStore();
 
     text_parser(parsed_text, vs);
 
-    t.equals(vs.var('input:query').toString(), 'query value');
-    t.equals(vs.var('input:category').toString(), 'category value');
-    t.equals(vs.var('input:housenumber').toString(), 'number value');
+    t.equals(vs.var('input:query').toString(), '13 this is 15 not a number 17');
+    t.false(vs.isset('input:housenumber'));
     t.equals(vs.var('input:street').toString(), 'street value');
-    t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value');
-    t.equals(vs.var('input:borough').toString(), 'borough value');
-    t.equals(vs.var('input:postcode').toString(), 'postalcode value');
-    t.equals(vs.var('input:locality').toString(), 'city value');
-    t.equals(vs.var('input:county').toString(), 'county value');
-    t.equals(vs.var('input:region').toString(), 'state value');
-    t.equals(vs.var('input:country').toString(), 'country value');
     t.end();
 
   });
diff --git a/test/unit/run.js b/test/unit/run.js
index 1cd8ab45..e93deb7d 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -41,6 +41,7 @@ var tests = [
   require('./query/reverse'),
   require('./query/search'),
   require('./query/search_original'),
+  require('./query/text_parser'),
   require('./sanitiser/_boundary_country'),
   require('./sanitiser/_flag_bool'),
   require('./sanitiser/_geo_common'),

From 8072c73d42b4a975e367953ea0555f00bd4c0fc9 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 12:12:37 -0400
Subject: [PATCH 55/82] removed unused dependency

---
 query/text_parser.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/query/text_parser.js b/query/text_parser.js
index 293c021e..c5b8da44 100644
--- a/query/text_parser.js
+++ b/query/text_parser.js
@@ -1,5 +1,4 @@
 var logger = require('pelias-logger').get('api');
-var _ = require('lodash');
 
 // all the address parsing logic
 function addParsedVariablesToQueryVariables( parsed_text, vs ){
@@ -78,7 +77,7 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){
   // In libpostal parlance, `house` is just a query term, not the house number.
   // This special case moves the query term to the house number field if there's a street,
   // there's no house number, and the query is parseable as an integer, then use the
-  // query as the house number and blank out the query.  
+  // query as the house number and blank out the query.
   if (shouldSetQueryIntoHouseNumber(vs)) {
     vs.var( 'input:housenumber', vs.var('input:query').toString());
     vs.unset( 'input:query' );

From 15b7c3fa57c2e0d2d78279a5f497bd385429dd80 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 12:23:43 -0400
Subject: [PATCH 56/82] uncommented tests

---
 test/unit/query/text_parser.js | 112 ++++++++++++++++-----------------
 1 file changed, 56 insertions(+), 56 deletions(-)

diff --git a/test/unit/query/text_parser.js b/test/unit/query/text_parser.js
index 274dc10c..34830c7f 100644
--- a/test/unit/query/text_parser.js
+++ b/test/unit/query/text_parser.js
@@ -10,62 +10,62 @@ module.exports.tests.interface = function(test, common) {
   });
 };
 
-// module.exports.tests.query = function(test, common) {
-//   test('parsed_text without properties should leave vs properties unset', function(t) {
-//     var parsed_text = {};
-//     var vs = new VariableStore();
-//
-//     text_parser(parsed_text, vs);
-//
-//     t.false(vs.isset('input:query'));
-//     t.false(vs.isset('input:category'));
-//     t.false(vs.isset('input:housenumber'));
-//     t.false(vs.isset('input:street'));
-//     t.false(vs.isset('input:neighbourhood'));
-//     t.false(vs.isset('input:borough'));
-//     t.false(vs.isset('input:postcode'));
-//     t.false(vs.isset('input:locality'));
-//     t.false(vs.isset('input:county'));
-//     t.false(vs.isset('input:region'));
-//     t.false(vs.isset('input:country'));
-//     t.end();
-//
-//   });
-//
-//   test('parsed_text without properties should leave vs properties unset', function(t) {
-//     var parsed_text = {
-//       query: 'query value',
-//       category: 'category value',
-//       number: 'number value',
-//       street: 'street value',
-//       neighbourhood: 'neighbourhood value',
-//       borough: 'borough value',
-//       postalcode: 'postalcode value',
-//       city: 'city value',
-//       county: 'county value',
-//       state: 'state value',
-//       country: 'country value'
-//     };
-//     var vs = new VariableStore();
-//
-//     text_parser(parsed_text, vs);
-//
-//     t.equals(vs.var('input:query').toString(), 'query value');
-//     t.equals(vs.var('input:category').toString(), 'category value');
-//     t.equals(vs.var('input:housenumber').toString(), 'number value');
-//     t.equals(vs.var('input:street').toString(), 'street value');
-//     t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value');
-//     t.equals(vs.var('input:borough').toString(), 'borough value');
-//     t.equals(vs.var('input:postcode').toString(), 'postalcode value');
-//     t.equals(vs.var('input:locality').toString(), 'city value');
-//     t.equals(vs.var('input:county').toString(), 'county value');
-//     t.equals(vs.var('input:region').toString(), 'state value');
-//     t.equals(vs.var('input:country').toString(), 'country value');
-//     t.end();
-//
-//   });
-//
-// };
+module.exports.tests.query = function(test, common) {
+  test('parsed_text without properties should leave vs properties unset', function(t) {
+    var parsed_text = {};
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.false(vs.isset('input:query'));
+    t.false(vs.isset('input:category'));
+    t.false(vs.isset('input:housenumber'));
+    t.false(vs.isset('input:street'));
+    t.false(vs.isset('input:neighbourhood'));
+    t.false(vs.isset('input:borough'));
+    t.false(vs.isset('input:postcode'));
+    t.false(vs.isset('input:locality'));
+    t.false(vs.isset('input:county'));
+    t.false(vs.isset('input:region'));
+    t.false(vs.isset('input:country'));
+    t.end();
+
+  });
+
+  test('parsed_text without properties should leave vs properties unset', function(t) {
+    var parsed_text = {
+      query: 'query value',
+      category: 'category value',
+      number: 'number value',
+      street: 'street value',
+      neighbourhood: 'neighbourhood value',
+      borough: 'borough value',
+      postalcode: 'postalcode value',
+      city: 'city value',
+      county: 'county value',
+      state: 'state value',
+      country: 'country value'
+    };
+    var vs = new VariableStore();
+
+    text_parser(parsed_text, vs);
+
+    t.equals(vs.var('input:query').toString(), 'query value');
+    t.equals(vs.var('input:category').toString(), 'category value');
+    t.equals(vs.var('input:housenumber').toString(), 'number value');
+    t.equals(vs.var('input:street').toString(), 'street value');
+    t.equals(vs.var('input:neighbourhood').toString(), 'neighbourhood value');
+    t.equals(vs.var('input:borough').toString(), 'borough value');
+    t.equals(vs.var('input:postcode').toString(), 'postalcode value');
+    t.equals(vs.var('input:locality').toString(), 'city value');
+    t.equals(vs.var('input:county').toString(), 'county value');
+    t.equals(vs.var('input:region').toString(), 'state value');
+    t.equals(vs.var('input:country').toString(), 'country value');
+    t.end();
+
+  });
+
+};
 
 module.exports.tests.housenumber_special_cases = function(test, common) {
   test('numeric query with street but no number should reassign query to housenumber', function(t) {

From b774cc617cfd21b7634b5c90b55a6109862fd4cb Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 16:00:49 -0400
Subject: [PATCH 57/82] fixed spelling error

---
 sanitiser/sanitizeAll.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sanitiser/sanitizeAll.js b/sanitiser/sanitizeAll.js
index ac31ddfe..86455a95 100644
--- a/sanitiser/sanitizeAll.js
+++ b/sanitiser/sanitizeAll.js
@@ -7,7 +7,7 @@ function sanitize( req, sanitizers, cb ){
   // (sanitized) input parameters
   req.clean = {};
 
-  // init erros and warnings arrays
+  // init errors and warnings arrays
   req.errors = [];
   req.warnings = [];
 

From f11bdfbfbb57cb2a878374246aa6894eeac592c2 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 16:01:53 -0400
Subject: [PATCH 58/82] removed unused dependency

---
 sanitiser/sanitizeAll.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/sanitiser/sanitizeAll.js b/sanitiser/sanitizeAll.js
index 86455a95..f4fd302a 100644
--- a/sanitiser/sanitizeAll.js
+++ b/sanitiser/sanitizeAll.js
@@ -1,6 +1,3 @@
-
-var check = require('check-types');
-
 function sanitize( req, sanitizers, cb ){
 
   // init an object to store clean

From 06310bd3b7d80df65b2c1d63bba7bcab7ec0c0d6 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 17:00:06 -0400
Subject: [PATCH 59/82] only initialize req.clean/errors/warnings if not
 initialized yet

also added tests for sanitizeAll
---
 sanitiser/sanitizeAll.js           |  12 +--
 test/unit/run.js                   |   1 +
 test/unit/sanitiser/sanitizeAll.js | 161 +++++++++++++++++++++++++++++
 3 files changed, 167 insertions(+), 7 deletions(-)
 create mode 100644 test/unit/sanitiser/sanitizeAll.js

diff --git a/sanitiser/sanitizeAll.js b/sanitiser/sanitizeAll.js
index f4fd302a..f6af363e 100644
--- a/sanitiser/sanitizeAll.js
+++ b/sanitiser/sanitizeAll.js
@@ -1,12 +1,10 @@
 function sanitize( req, sanitizers, cb ){
+  // init an object to store clean (sanitized) input parameters if not initialized
+  req.clean = req.clean || {};
 
-  // init an object to store clean
-  // (sanitized) input parameters
-  req.clean = {};
-
-  // init errors and warnings arrays
-  req.errors = [];
-  req.warnings = [];
+  // init errors and warnings arrays if not initialized
+  req.errors = req.errors || [];
+  req.warnings = req.warnings || [];
 
   // source of input parameters
   // (in this case from the GET querystring params)
diff --git a/test/unit/run.js b/test/unit/run.js
index e93deb7d..af155991 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -63,6 +63,7 @@ var tests = [
   require('./sanitiser/autocomplete'),
   require('./sanitiser/place'),
   require('./sanitiser/reverse'),
+  require('./sanitiser/sanitizeAll'),
   require('./sanitiser/search'),
   require('./sanitiser/search_fallback'),
   require('./sanitiser/wrap'),
diff --git a/test/unit/sanitiser/sanitizeAll.js b/test/unit/sanitiser/sanitizeAll.js
new file mode 100644
index 00000000..3d78e599
--- /dev/null
+++ b/test/unit/sanitiser/sanitizeAll.js
@@ -0,0 +1,161 @@
+var sanitizeAll = require('../../../sanitiser/sanitizeAll');
+
+module.exports.tests = {};
+
+module.exports.tests.all = function(test, common) {
+  test('req.clean/errors/warnings should be initialized when they are not', function(t) {
+    var req = {};
+    var sanitizers = [
+      function() {
+        req.clean.a = 'first sanitizer';
+        return {
+          errors: ['error 1', 'error 2'],
+          warnings: ['warning 1', 'warning 2']
+        };
+      },
+      function() {
+        req.clean.b = 'second sanitizer';
+        return {
+          errors: ['error 3'],
+          warnings: ['warning 3']
+        };
+      }
+    ];
+
+    var expected_req = {
+      clean: {
+        a: 'first sanitizer',
+        b: 'second sanitizer'
+      },
+      errors: ['error 1', 'error 2', 'error 3'],
+      warnings: ['warning 1', 'warning 2', 'warning 3']
+    };
+
+    sanitizeAll(req, sanitizers, function(){
+      t.deepEquals(req, expected_req);
+      t.end();
+    });
+
+  });
+
+  test('req.clean/errors/warnings should not be initialized when they already have been', function(t) {
+    var req = {
+      clean: {
+        alreadyInitialized: true
+      },
+      errors: ['pre-existing error'],
+      warnings: ['pre-existing warning']
+    };
+
+    var sanitizers = [
+      function() {
+        req.clean.a = 'first sanitizer';
+        return {
+          errors: ['error 1', 'error 2'],
+          warnings: ['warning 1', 'warning 2']
+        };
+      },
+      function() {
+        req.clean.b = 'second sanitizer';
+        return {
+          errors: ['error 3'],
+          warnings: ['warning 3']
+        };
+      }
+    ];
+
+    var expected_req = {
+      clean: {
+        alreadyInitialized: true,
+        a: 'first sanitizer',
+        b: 'second sanitizer'
+      },
+      errors: ['pre-existing error', 'error 1', 'error 2', 'error 3'],
+      warnings: ['pre-existing warning', 'warning 1', 'warning 2', 'warning 3']
+    };
+
+    sanitizeAll(req, sanitizers, function(){
+      t.deepEquals(req, expected_req);
+      t.end();
+    });
+
+  });
+
+  test('req.query should be passed to individual sanitizers when available', function(t) {
+    var req = {
+      query: {
+        value: 'query value'
+      }
+    };
+    var sanitizers = [
+      function(params) {
+        req.clean.query = params;
+        return {
+          errors: [],
+          warnings: []
+        };
+      }
+    ];
+
+    var expected_req = {
+      query: {
+        value: 'query value'
+      },
+      clean: {
+        query: {
+          value: 'query value'
+        }
+      },
+      errors: [],
+      warnings: []
+    };
+
+    sanitizeAll(req, sanitizers, function(){
+      t.deepEquals(req, expected_req);
+      t.end();
+    });
+
+  });
+
+  test('an empty object should be passed to individual sanitizers when req.query is unavailable', function(t) {
+    var req = {};
+    var sanitizers = [
+      function(params) {
+        if (Object.keys(params).length === 0) {
+          req.clean.empty_object_was_passed = true;
+        }
+
+        return {
+          errors: [],
+          warnings: []
+        };
+      }
+    ];
+
+    var expected_req = {
+      clean: {
+        empty_object_was_passed: true
+      },
+      errors: [],
+      warnings: []
+    };
+
+    sanitizeAll(req, sanitizers, function(){
+      t.deepEquals(req, expected_req);
+      t.end();
+    });
+
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SANITIZE sanitizeAll ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

From cc5c9f8468b85b36d864f8e996f0d5735c7555fc Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 8 Sep 2016 19:36:21 -0400
Subject: [PATCH 60/82] disable calling GeodisambiguationQuery

---
 controller/search.js           |  9 ++++++++-
 query/search.js                |  3 ++-
 test/unit/controller/search.js | 27 +++++++++++++++++++++++++++
 test/unit/query/search.js      | 21 +++++++++++----------
 4 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/controller/search.js b/controller/search.js
index b4124acf..30a6cd1c 100644
--- a/controller/search.js
+++ b/controller/search.js
@@ -31,11 +31,18 @@ function setup( config, backend, query ){
     // log clean parameters for stats
     logger.info('[req]', 'endpoint=' + req.path, cleanOutput);
 
+    var query_body = query(req.clean);
+
+    // if there's no query to call ES with, skip the service
+    if (_.isUndefined(query_body)) {
+      return next();
+    }
+
     // backend command
     var cmd = {
       index: config.indexName,
       searchType: 'dfs_query_then_fetch',
-      body: query( req.clean )
+      body: query_body
     };
 
     logger.debug( '[ES req]', cmd );
diff --git a/query/search.js b/query/search.js
index b01fa6a1..2ab51df7 100644
--- a/query/search.js
+++ b/query/search.js
@@ -121,7 +121,8 @@ function generateQuery( clean ){
 
 function getQuery(vs) {
   if (isSingleFieldGeoambiguity(vs) && !hasQueryOrAddress(vs)) {
-    return geodisambiguationQuery.render(vs);
+    // return `undefined` for now until we exorcise the geodisambiguation demons
+    return;
   } else {
     return fallbackQuery.render(vs);
   }
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 0c8e40ae..408be007 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -1,6 +1,7 @@
 var setup = require('../../../controller/search'),
     mockBackend = require('../mock/backend'),
     mockQuery = require('../mock/query');
+var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
 
@@ -193,6 +194,32 @@ module.exports.tests.existing_results = function(test, common) {
 
 };
 
+module.exports.tests.undefined_query = function(test, common) {
+  test('query returning undefined should not call service', function(t) {
+    // a function that returns undefined
+    var query = function() {
+      return;
+    };
+
+    var search_service_was_called = false;
+
+    var controller = proxyquire('../../../controller/search', {
+      '../service/search': function() {
+        search_service_was_called = true;
+        throw new Error('search service should not have been called');
+      }
+    })(undefined, undefined, query);
+
+    var next = function() {
+      t.notOk(search_service_was_called, 'should have returned before search service was called');
+      t.end();
+    };
+
+    controller({}, {}, next);
+
+  });
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index 86e582b7..29fa4d61 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -150,19 +150,20 @@ module.exports.tests.query = function(test, common) {
 
   });
 
-  test('parsed_text with single admin field should use GeodisambiguationQuery', function(t) {
-    var clean = {
-      parsed_text: {
-        neighbourhood: 'neighbourhood value'
-      }
-    };
+  test('parsed_text with single admin field should return undefined', function(t) {
+    ['neighbourhood', 'borough', 'city', 'county', 'state', 'country'].forEach(function(placeType) {
+        var clean = {
+          parsed_text: {}
+        };
 
-    var query = generate(clean);
+        clean.parsed_text[placeType] = placeType + ' value';
 
-    var compiled = JSON.parse(JSON.stringify(query));
-    var expected = require('../fixture/search_geodisambiguation');
+        var query = generate(clean);
+
+        t.equals(query, undefined, 'geodisambiguationQuery');
+
+      });
 
-    t.deepEqual(compiled, expected, 'geodisambiguationQuery');
     t.end();
 
   });

From 768843b0fb55eafd9dbf05f7a2aa24345b527e03 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 14 Sep 2016 13:54:50 -0400
Subject: [PATCH 61/82] delete `clean.parsed_text` if falling back to addressit

---
 sanitiser/_text_addressit.js           | 3 +++
 test/unit/sanitiser/_text_addressit.js | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/sanitiser/_text_addressit.js b/sanitiser/_text_addressit.js
index 5578c4b1..04fca21a 100644
--- a/sanitiser/_text_addressit.js
+++ b/sanitiser/_text_addressit.js
@@ -21,6 +21,9 @@ function sanitize( raw, clean ){
     // valid text
     clean.text = raw.text;
 
+    // remove anything that may have been parsed before
+    delete clean.parsed_text;
+
     // parse text with query parser
     var parsed_text = parse(clean.text);
     if (check.assigned(parsed_text)) {
diff --git a/test/unit/sanitiser/_text_addressit.js b/test/unit/sanitiser/_text_addressit.js
index 6a67a3db..c6f19660 100644
--- a/test/unit/sanitiser/_text_addressit.js
+++ b/test/unit/sanitiser/_text_addressit.js
@@ -79,6 +79,7 @@ module.exports.tests.text_parser = function(test, common) {
       text: 'yugolsavia'
     };
     var clean = {};
+    clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
       text: 'yugolsavia'
@@ -97,6 +98,7 @@ module.exports.tests.text_parser = function(test, common) {
       text: 'small town'
     };
     var clean = {};
+    clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
       text: 'small town'
@@ -115,6 +117,7 @@ module.exports.tests.text_parser = function(test, common) {
       text: '123 main'
     };
     var clean = {};
+    clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
       text: '123 main'
@@ -133,6 +136,7 @@ module.exports.tests.text_parser = function(test, common) {
       text: 'main 123'
     };
     var clean = {};
+    clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
       text: 'main 123'
@@ -151,6 +155,7 @@ module.exports.tests.text_parser = function(test, common) {
       text: 'main particle new york'
     };
     var clean = {};
+    clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
       text: 'main particle new york'

From c841ed8121f105897fe07a3628914035640442a6 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Wed, 14 Sep 2016 16:18:17 -0400
Subject: [PATCH 62/82] limited fallbackQuery usage to analysis with `street`

---
 query/search.js           | 17 ++-----
 test/unit/query/search.js | 96 ++++++++++++++++++++++++++++++---------
 2 files changed, 77 insertions(+), 36 deletions(-)

diff --git a/query/search.js b/query/search.js
index 2ab51df7..64d6f67d 100644
--- a/query/search.js
+++ b/query/search.js
@@ -120,25 +120,14 @@ function generateQuery( clean ){
 }
 
 function getQuery(vs) {
-  if (isSingleFieldGeoambiguity(vs) && !hasQueryOrAddress(vs)) {
-    // return `undefined` for now until we exorcise the geodisambiguation demons
-    return;
-  } else {
+  if (hasStreet(vs)) {
     return fallbackQuery.render(vs);
   }
 
 }
 
-function isSingleFieldGeoambiguity(vs) {
-  return ['neighbourhood', 'borough', 'locality', 'county', 'region', 'country'].filter(function(layer) {
-    return vs.isset('input:' + layer);
-  }).length === 1;
-}
-
-function hasQueryOrAddress(vs) {
-  return ['housenumber', 'street', 'query', 'category'].filter(function(layer) {
-    return vs.isset('input:' + layer);
-  }).length > 0;
+function hasStreet(vs) {
+  return vs.isset('input:street');
 }
 
 module.exports = generateQuery;
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index 29fa4d61..c758a23e 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -12,15 +12,21 @@ module.exports.tests.interface = function(test, common) {
 
 module.exports.tests.query = function(test, common) {
   test('valid search + focus + bbox', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
+      text: 'test',
+      querySize: 10,
       'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
       'boundary.rect.min_lat': 47.47,
       'boundary.rect.max_lon': -61.84,
       'boundary.rect.max_lat': 11.51,
       'boundary.rect.min_lon': -103.16,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_bbox');
@@ -30,14 +36,20 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('valid search + bbox', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
+      text: 'test',
+      querySize: 10,
       'boundary.rect.min_lat': 47.47,
       'boundary.rect.max_lon': -61.84,
       'boundary.rect.max_lat': 11.51,
       'boundary.rect.min_lon': -103.16,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_bbox');
@@ -47,10 +59,15 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('valid lingustic-only search', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_only');
@@ -60,11 +77,16 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('search search + focus', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus');
@@ -74,14 +96,19 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('search search + viewport', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49136,
       'focus.viewport.max_lat': 30.49136,
       'focus.viewport.min_lon': -87.50622,
       'focus.viewport.max_lon': -77.50622,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_viewport');
@@ -93,14 +120,19 @@ module.exports.tests.query = function(test, common) {
   // viewport scale sizing currently disabled.
   // ref: https://github.com/pelias/api/pull/388
   test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49135,
       'focus.viewport.max_lat': 28.49137,
       'focus.viewport.min_lon': -87.50622,
       'focus.viewport.max_lon': -87.50624,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
@@ -110,11 +142,16 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('search search + focus on null island', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       'focus.point.lat': 0, 'focus.point.lon': 0,
       layers: ['test']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_null_island');
@@ -169,11 +206,16 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('valid boundary.country search', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       text: 'test', querySize: 10,
       layers: ['test'],
       'boundary.country': 'ABC'
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_boundary_country');
@@ -183,10 +225,15 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('valid sources filter', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       'text': 'test',
       'sources': ['test_source']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_source_filtering');
@@ -196,10 +243,15 @@ module.exports.tests.query = function(test, common) {
   });
 
   test('categories filter', function(t) {
-    var query = generate({
+    var clean = {
+      parsed_text: {
+        street: 'street value'
+      },
       'text': 'test',
       'categories': ['retail','food']
-    });
+    };
+
+    var query = generate(clean);
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_category_filtering');

From d4ed96de0cc186451c19d6b7d81037cb6355cd26 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 15 Sep 2016 16:44:19 -0400
Subject: [PATCH 63/82] added explicit `return undefined` + comments

---
 query/search.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/query/search.js b/query/search.js
index 64d6f67d..33847da5 100644
--- a/query/search.js
+++ b/query/search.js
@@ -124,6 +124,10 @@ function getQuery(vs) {
     return fallbackQuery.render(vs);
   }
 
+  // returning undefined is a signal to a later step that the addressit-parsed
+  // query should be queried for
+  return undefined;
+
 }
 
 function hasStreet(vs) {

From cf1aeec4e15eb87e0514f10837a544055cef3033 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 16 Sep 2016 10:32:29 -0400
Subject: [PATCH 64/82] fixed tests for latest query changes

---
 test/unit/fixture/search_fallback.js | 96 +++++++++++++++++++++++++++-
 1 file changed, 95 insertions(+), 1 deletion(-)

diff --git a/test/unit/fixture/search_fallback.js b/test/unit/fixture/search_fallback.js
index 12b617a3..f7983da9 100644
--- a/test/unit/fixture/search_fallback.js
+++ b/test/unit/fixture/search_fallback.js
@@ -109,10 +109,97 @@ module.exports = {
                           'address_parts.street': 'street value'
                         }
                       },
+                      {
+                        'multi_match': {
+                          'query': 'neighbourhood value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.neighbourhood',
+                            'parent.neighbourhood_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'borough value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.borough',
+                            'parent.borough_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'city value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.locality',
+                            'parent.locality_a',
+                            'parent.localadmin',
+                            'parent.localadmin_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'county value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.county',
+                            'parent.county_a',
+                            'parent.macrocounty',
+                            'parent.macrocounty_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'state value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.region',
+                            'parent.region_a',
+                            'parent.macroregion',
+                            'parent.macroregion_a'
+                          ]
+                        }
+                      },
+                      {
+                        'multi_match': {
+                          'query': 'country value',
+                          'type': 'phrase',
+                          'fields': [
+                            'parent.country',
+                            'parent.country_a',
+                            'parent.dependency',
+                            'parent.dependency_a'
+                          ]
+                        }
+                      }
+                    ],
+                    'should': [
                       {
                         'match_phrase': {
                           'address_parts.zip': 'postalcode value'
                         }
+                      }
+                    ],
+                    'filter': {
+                      'term': {
+                        'layer': 'address'
+                      }
+                    }
+                  }
+                },
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
                       },
                       {
                         'multi_match': {
@@ -183,9 +270,16 @@ module.exports = {
                         }
                       }
                     ],
+                    'should': [
+                      {
+                        'match_phrase': {
+                          'address_parts.zip': 'postalcode value'
+                        }
+                      }
+                    ],
                     'filter': {
                       'term': {
-                        'layer': 'address'
+                        'layer': 'street'
                       }
                     }
                   }

From df2ac8c91b69605749f3178cf573b0c1b691176a Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 16 Sep 2016 10:34:00 -0400
Subject: [PATCH 65/82] added `street` to trimByGranularity

---
 middleware/trimByGranularity.js           |  1 +
 test/unit/middleware/trimByGranularity.js | 37 +++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/middleware/trimByGranularity.js b/middleware/trimByGranularity.js
index d5ff528b..da4ef4f9 100644
--- a/middleware/trimByGranularity.js
+++ b/middleware/trimByGranularity.js
@@ -16,6 +16,7 @@ var _ = require('lodash');
 var layers = [
   'venue',
   'address',
+  'street',
   'neighbourhood',
   'borough',
   'locality',
diff --git a/test/unit/middleware/trimByGranularity.js b/test/unit/middleware/trimByGranularity.js
index fdb1e839..1eb65206 100644
--- a/test/unit/middleware/trimByGranularity.js
+++ b/test/unit/middleware/trimByGranularity.js
@@ -20,6 +20,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'venue 1', _matched_queries: ['fallback.venue'] },
         { name: 'venue 2', _matched_queries: ['fallback.venue'] },
         { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
         { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
@@ -55,6 +56,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'address 1', _matched_queries: ['fallback.address'] },
         { name: 'address 2', _matched_queries: ['fallback.address'] },
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
         { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
@@ -83,6 +85,41 @@ module.exports.tests.trimByGranularity = function(test, common) {
     testIt();
   });
 
+  test('all records with fallback.* matched_queries name should retain only streets when they are most granular', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'street 1', _matched_queries: ['fallback.street'] },
+        { name: 'street 2', _matched_queries: ['fallback.street'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'street 1', _matched_queries: ['fallback.street'] },
+      { name: 'street 2', _matched_queries: ['fallback.street'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only street records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
+
   test('all records with fallback.* matched_queries name should retain only neighbourhoods when they are most granular', function(t) {
     var req = { clean: {} };
 

From 5790026adde88ce9a5f613b5fe8df5f3a1984b19 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 16 Sep 2016 14:29:53 -0400
Subject: [PATCH 66/82] trigger rebuild

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 8c88388c..b93b40bf 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "pelias/query#reorganize-queries-for-scoring",
+    "pelias-query": "pelias/query#reorganize-queries-for-scoring", 
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"

From 80e9fc06e4192d9c22cfb56ec121116685e1f179 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 16 Sep 2016 14:30:16 -0400
Subject: [PATCH 67/82] trigger rebuild

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index b93b40bf..8c88388c 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "pelias/query#reorganize-queries-for-scoring", 
+    "pelias-query": "pelias/query#reorganize-queries-for-scoring",
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"

From 24d81b754f913cb4f04e026a08e5fe172c47199e Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Fri, 16 Sep 2016 15:38:11 -0400
Subject: [PATCH 68/82] updated fixtures for latest query

---
 package.json                                  |  2 +-
 test/unit/fixture/search_boundary_country.js  | 21 ++++++++++++++++++-
 test/unit/fixture/search_linguistic_bbox.js   | 21 ++++++++++++++++++-
 test/unit/fixture/search_linguistic_focus.js  | 21 ++++++++++++++++++-
 .../fixture/search_linguistic_focus_bbox.js   | 21 ++++++++++++++++++-
 .../search_linguistic_focus_null_island.js    | 21 ++++++++++++++++++-
 test/unit/fixture/search_linguistic_only.js   | 21 ++++++++++++++++++-
 .../fixture/search_linguistic_viewport.js     | 21 ++++++++++++++++++-
 ...search_linguistic_viewport_min_diagonal.js | 21 ++++++++++++++++++-
 .../fixture/search_with_category_filtering.js | 21 ++++++++++++++++++-
 .../fixture/search_with_source_filtering.js   | 21 ++++++++++++++++++-
 11 files changed, 201 insertions(+), 11 deletions(-)

diff --git a/package.json b/package.json
index 8c88388c..cc3ac6df 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "pelias/query#reorganize-queries-for-scoring",
+    "pelias-query": "8.6.0",
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"
diff --git a/test/unit/fixture/search_boundary_country.js b/test/unit/fixture/search_boundary_country.js
index ea653da8..96fb4170 100644
--- a/test/unit/fixture/search_boundary_country.js
+++ b/test/unit/fixture/search_boundary_country.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_bbox.js b/test/unit/fixture/search_linguistic_bbox.js
index e6582e29..46e2fccd 100644
--- a/test/unit/fixture/search_linguistic_bbox.js
+++ b/test/unit/fixture/search_linguistic_bbox.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_focus.js b/test/unit/fixture/search_linguistic_focus.js
index 6af7efcf..b2e577b1 100644
--- a/test/unit/fixture/search_linguistic_focus.js
+++ b/test/unit/fixture/search_linguistic_focus.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_focus_bbox.js b/test/unit/fixture/search_linguistic_focus_bbox.js
index 4126d479..1f61dc6d 100644
--- a/test/unit/fixture/search_linguistic_focus_bbox.js
+++ b/test/unit/fixture/search_linguistic_focus_bbox.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_focus_null_island.js b/test/unit/fixture/search_linguistic_focus_null_island.js
index 9e7b299c..fc47bc4e 100644
--- a/test/unit/fixture/search_linguistic_focus_null_island.js
+++ b/test/unit/fixture/search_linguistic_focus_null_island.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_only.js b/test/unit/fixture/search_linguistic_only.js
index 37767375..caa4aefa 100644
--- a/test/unit/fixture/search_linguistic_only.js
+++ b/test/unit/fixture/search_linguistic_only.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_viewport.js b/test/unit/fixture/search_linguistic_viewport.js
index 37767375..caa4aefa 100644
--- a/test/unit/fixture/search_linguistic_viewport.js
+++ b/test/unit/fixture/search_linguistic_viewport.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
index 37767375..caa4aefa 100644
--- a/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
+++ b/test/unit/fixture/search_linguistic_viewport_min_diagonal.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_with_category_filtering.js b/test/unit/fixture/search_with_category_filtering.js
index 9aa12a66..9913b19c 100644
--- a/test/unit/fixture/search_with_category_filtering.js
+++ b/test/unit/fixture/search_with_category_filtering.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {
diff --git a/test/unit/fixture/search_with_source_filtering.js b/test/unit/fixture/search_with_source_filtering.js
index 0fdbdc24..78889325 100644
--- a/test/unit/fixture/search_with_source_filtering.js
+++ b/test/unit/fixture/search_with_source_filtering.js
@@ -5,7 +5,26 @@ module.exports = {
         'filtered': {
           'query': {
             'bool': {
-              'should': []
+              'should': [
+                {
+                  'bool': {
+                    '_name': 'fallback.street',
+                    'must': [
+                      {
+                        'match_phrase': {
+                          'address_parts.street': 'street value'
+                        }
+                      }
+                    ],
+                    'should': [],
+                    'filter': {
+                      'term': {
+                        'layer': 'street'
+                      }
+                    }
+                  }
+                }
+              ]
             }
           },
           'filter': {

From da43bf46953b56cc8743ce59c99ed3b4cb2827db Mon Sep 17 00:00:00 2001
From: greenkeeperio-bot <support@greenkeeper.io>
Date: Sun, 18 Sep 2016 14:03:17 -0400
Subject: [PATCH 69/82] chore(package): update geojson to version 0.4.0

https://greenkeeper.io/
---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 31c8d36f..a06f4282 100644
--- a/package.json
+++ b/package.json
@@ -42,7 +42,7 @@
     "express": "^4.8.8",
     "express-http-proxy": "^0.7.0",
     "extend": "3.0.0",
-    "geojson": "^0.3.0",
+    "geojson": "^0.4.0",
     "geojson-extent": "^0.3.1",
     "geolib": "^2.0.18",
     "geopipes-elasticsearch-backend": "^0.2.0",

From 9fd19242e70add9fa152ad7b3fc14027c1f0e612 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 13:56:37 -0400
Subject: [PATCH 70/82] make existing confidence score only handle the original
 query type

---
 middleware/confidenceScore.js                 |   5 +-
 middleware/confidenceScoreFallback.js         | 255 ++++++++++++++++++
 .../middleware/confidenceScoreFallback.js     | 182 +++++++++++++
 test/unit/mock/search_query.js                |  10 +
 4 files changed, 450 insertions(+), 2 deletions(-)
 create mode 100644 middleware/confidenceScoreFallback.js
 create mode 100644 test/unit/middleware/confidenceScoreFallback.js
 create mode 100644 test/unit/mock/search_query.js

diff --git a/middleware/confidenceScore.js b/middleware/confidenceScore.js
index 2e9eb1c6..8f5b61fb 100644
--- a/middleware/confidenceScore.js
+++ b/middleware/confidenceScore.js
@@ -25,9 +25,10 @@ function setup(peliasConfig) {
 }
 
 function computeScores(req, res, next) {
-  // do nothing if no result data set
+  // do nothing if no result data set or if query is not of the original variety
   if (check.undefined(req.clean) || check.undefined(res) ||
-      check.undefined(res.data) || check.undefined(res.meta)) {
+      check.undefined(res.data) || check.undefined(res.meta) ||
+      res.meta.query_type !== 'original') {
     return next();
   }
 
diff --git a/middleware/confidenceScoreFallback.js b/middleware/confidenceScoreFallback.js
new file mode 100644
index 00000000..cead1a35
--- /dev/null
+++ b/middleware/confidenceScoreFallback.js
@@ -0,0 +1,255 @@
+/**
+ *
+ * Basic confidence score should be computed and returned for each item in the results.
+ * The score should range between 0-1, and take into consideration as many factors as possible.
+ *
+ * Some factors to consider:
+ *
+ * - number of results from ES
+ * - fallback status (aka layer match between expected and actual)
+ */
+
+var check = require('check-types');
+
+function setup() {
+  return computeScores;
+}
+
+function computeScores(req, res, next) {
+  // do nothing if no result data set or if the query is not of the fallback variety
+  // later add disambiguation to this list
+  if (check.undefined(req.clean) || check.undefined(res) ||
+      check.undefined(res.data) || check.undefined(res.meta) ||
+      res.meta.query_type !== 'fallback') {
+    return next();
+  }
+
+  // loop through data items and determine confidence scores
+  res.data = res.data.map(computeConfidenceScore.bind(null, req));
+
+  next();
+}
+
+/**
+ * Check all types of things to determine how confident we are that this result
+ * is correct.
+ *
+ * @param {object} req
+ * @param {object} hit
+ * @returns {object}
+ */
+function computeConfidenceScore(req, hit) {
+  var dealBreakers = checkForDealBreakers(req, hit);
+  if (dealBreakers) {
+    hit.confidence = 0.5;
+    return hit;
+  }
+
+  var checkCount = 3;
+  hit.confidence = 0;
+
+  if (RELATIVE_SCORES) {
+    checkCount += 2;
+    hit.confidence += checkDistanceFromMean(hit._score, mean, stdev);
+    hit.confidence += computeZScore(hit._score, mean, stdev);
+  }
+  hit.confidence += checkName(req.clean.text, req.clean.parsed_text, hit);
+  hit.confidence += checkQueryType(req.clean.parsed_text, hit);
+  hit.confidence += checkAddress(req.clean.parsed_text, hit);
+
+  // TODO: look at categories and location
+
+  hit.confidence /= checkCount;
+  hit.confidence = Number((hit.confidence).toFixed(3));
+
+  return hit;
+}
+
+/*
+ * Check for clearly mismatching properties in a result
+ * zip code and state (region) are currently checked if present
+ *
+ * @param {object|undefined} text
+ * @param {object} hit
+ * @returns {bool}
+ */
+function checkForDealBreakers(req, hit) {
+  if (check.undefined(req.clean.parsed_text)) {
+    return false;
+  }
+
+  if (check.assigned(req.clean.parsed_text.state) && hit.parent.region_a && req.clean.parsed_text.state !== hit.parent.region_a[0]) {
+    logger.debug('[confidence][deal-breaker]: state !== region_a');
+    return true;
+  }
+
+  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) &&
+      req.clean.parsed_text.postalcode !== hit.address_parts.zip) {
+    return true;
+  }
+}
+
+/**
+ * Check how statistically significant the score of this result is
+ * given mean and standard deviation
+ *
+ * @param {number} score
+ * @param {number} mean
+ * @param {number} stdev
+ * @returns {number}
+ */
+function checkDistanceFromMean(score, mean, stdev) {
+  return (score - mean) > stdev ? 1 : 0;
+}
+
+/**
+ * Compare text string or name component of parsed_text against
+ * default name in result
+ *
+ * @param {string} text
+ * @param {object|undefined} parsed_text
+ * @param {object} hit
+ * @returns {number}
+ */
+function checkName(text, parsed_text, hit) {
+  // parsed_text name should take precedence if available since it's the cleaner name property
+  if (check.assigned(parsed_text) && check.assigned(parsed_text.name) &&
+    hit.name.default.toLowerCase() === parsed_text.name.toLowerCase()) {
+    return 1;
+  }
+
+  // if no parsed_text check the text value as provided against result's default name
+  if (hit.name.default.toLowerCase() === text.toLowerCase()) {
+    return 1;
+  }
+
+  // if no matches detected, don't judge too harshly since it was a longshot anyway
+  return 0.7;
+}
+
+/**
+ * text being set indicates the query was for an address
+ * check if house number was specified and found in result
+ *
+ * @param {object|undefined} text
+ * @param {object} hit
+ * @returns {number}
+ */
+function checkQueryType(text, hit) {
+  if (check.assigned(text) && check.assigned(text.number) &&
+      (check.undefined(hit.address_parts) ||
+      (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) {
+    return 0;
+  }
+  return 1;
+}
+
+/**
+ * Determine the quality of the property match
+ *
+ * @param {string|number|undefined|null} textProp
+ * @param {string|number|undefined|null} hitProp
+ * @param {boolean} expectEnriched
+ * @returns {number}
+ */
+function propMatch(textProp, hitProp, expectEnriched) {
+
+  // both missing, but expect to have enriched value in result => BAD
+  if (check.undefined(textProp) && check.undefined(hitProp) && check.assigned(expectEnriched)) { return 0; }
+
+  // both missing, and no enrichment expected => GOOD
+  if (check.undefined(textProp) && check.undefined(hitProp)) { return 1; }
+
+  // text has it, result doesn't => BAD
+  if (check.assigned(textProp) && check.undefined(hitProp)) { return 0; }
+
+  // text missing, result has it, and enrichment is expected => GOOD
+  if (check.undefined(textProp) && check.assigned(hitProp) && check.assigned(expectEnriched)) { return 1; }
+
+  // text missing, result has it, enrichment not desired => 50/50
+  if (check.undefined(textProp) && check.assigned(hitProp)) { return 0.5; }
+
+  // both present, values match => GREAT
+  if (check.assigned(textProp) && check.assigned(hitProp) &&
+      textProp.toString().toLowerCase() === hitProp.toString().toLowerCase()) { return 1; }
+
+  // ¯\_(ツ)_/¯
+  return 0.7;
+}
+
+/**
+ * Check various parts of the parsed text address
+ * against the results
+ *
+ * @param {object} text
+ * @param {string|number} [text.number]
+ * @param {string} [text.street]
+ * @param {string} [text.postalcode]
+ * @param {string} [text.state]
+ * @param {string} [text.country]
+ * @param {object} hit
+ * @param {object} [hit.address_parts]
+ * @param {string|number} [hit.address_parts.number]
+ * @param {string} [hit.address_parts.street]
+ * @param {string|number} [hit.address_parts.zip]
+ * @param {Array} [hit.parent.region_a]
+ * @param {Array} [hit.parent.country_a]
+ * @returns {number}
+ */
+function checkAddress(text, hit) {
+  var checkCount = 5;
+  var res = 0;
+
+  if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) {
+    res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false);
+    res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false);
+    res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true);
+    res += propMatch(text.state, (hit.parent.region_a ? hit.parent.region_a[0] : null), true);
+    res += propMatch(text.country, (hit.parent.country_a ? hit.parent.country_a[0] :null), true);
+
+    res /= checkCount;
+  }
+  else {
+    res = 1;
+  }
+
+  return res;
+}
+
+/**
+ * z-scores have an effective range of -3.00 to +3.00.
+ * An average z-score is ZERO.
+ * A negative z-score indicates that the item/element is below
+ * average and a positive z-score means that the item/element
+ * in above average. When teachers say they are going to "curve"
+ * the test, they do this by computing z-scores for the students' test scores.
+ *
+ * @param {number} score
+ * @param {number} mean
+ * @param {number} stdev
+ * @returns {number}
+ */
+function computeZScore(score, mean, stdev) {
+  if (stdev < 0.01) {
+    return 0;
+  }
+  // because the effective range of z-scores is -3.00 to +3.00
+  // add 10 to ensure a positive value, and then divide by 10+3+3
+  // to further normalize to %-like result
+  return (((score - mean) / (stdev)) + 10) / 16;
+}
+
+/**
+ * Computes standard deviation given an array of values
+ *
+ * @param {Array} scores
+ * @returns {number}
+ */
+function computeStandardDeviation(scores) {
+  var stdev = stats.stdev(scores);
+  // if stdev is low, just consider it 0
+  return (stdev < 0.01) ? 0 : stdev;
+}
+
+
+module.exports = setup;
diff --git a/test/unit/middleware/confidenceScoreFallback.js b/test/unit/middleware/confidenceScoreFallback.js
new file mode 100644
index 00000000..a0b4de6f
--- /dev/null
+++ b/test/unit/middleware/confidenceScoreFallback.js
@@ -0,0 +1,182 @@
+var confidenceScore = require('../../../middleware/confidenceScore')();
+
+module.exports.tests = {};
+
+module.exports.tests.confidenceScore = function(test, common) {
+
+  test('empty res and req should not throw exception', function(t) {
+    function testIt() {
+      confidenceScore({}, {}, function() {});
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.end();
+  });
+
+  test('res.results without parsed_text should not throw exception', function(t) {
+    var req = {};
+    var res = {
+      data: [{
+        name: 'foo'
+      }],
+      meta: [10]
+    };
+
+    function testIt() {
+      confidenceScore(req, res, function() {});
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.end();
+  });
+
+  test('hit without address should not error', function(t) {
+    var req = {
+      clean: {
+        text: 'test name3',
+        parsed_text: {
+          postalcode: 12345
+        }
+      }
+    };
+    var res = {
+      data: [{
+        name: {
+          default: 'foo'
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'original'
+      }
+    };
+
+    function testIt() {
+      confidenceScore(req, res, function() {});
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown with no address');
+    t.end();
+  });
+
+
+  test('res.results without parsed_text should not throw exception', function(t) {
+    var req = {
+      clean: { text: 'test name1' }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1'],
+          region: ['state1'],
+          county: ['city1']
+        }
+      }, {
+        _score: 20,
+        value: 2,
+        center_point: { lat: 100.2, lon: -51.5 },
+        name: { default: 'test name2' },
+        parent: {
+          country: ['country2'],
+          region: ['state2'],
+          county: ['city2']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'original'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.equal(res.data[0].confidence, 0.6, 'score was set');
+    t.end();
+  });
+
+  test('undefined region fields should be handled gracefully', function(t) {
+    var req = {
+      clean: {
+        text: '123 Main St, City, NM',
+        parsed_text: {
+          number: 123,
+          street: 'Main St',
+          state: 'NM'
+        }
+      }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1'],
+          region: undefined,
+          region_a: undefined,
+          county: ['city1']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'original'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.equal(res.data[0].confidence, 0.28, 'score was set');
+    t.end();
+  });
+
+  test('should only work for original query_type', function(t) {
+    var req = {
+      clean: {
+        text: '123 Main St, City, NM',
+        parsed_text: {
+          number: 123,
+          street: 'Main St',
+          state: 'NM'
+        }
+      }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1'],
+          region: undefined,
+          region_a: undefined,
+          county: ['city1']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'fallback'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.false(res.data[0].hasOwnProperty('confidence'), 'score was not set');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('[middleware] confidenceScore: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/test/unit/mock/search_query.js b/test/unit/mock/search_query.js
new file mode 100644
index 00000000..2a5f21fd
--- /dev/null
+++ b/test/unit/mock/search_query.js
@@ -0,0 +1,10 @@
+
+function setup(){
+  return query;
+}
+
+function query( clean ){
+  return clean;
+}
+
+module.exports = setup;
\ No newline at end of file

From fd3ec97ad01718c850a9b3e379451ba5c57fe392 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 13:58:09 -0400
Subject: [PATCH 71/82] add new confidence score computation for fallback query
 type

---
 middleware/confidenceScoreFallback.js         | 247 ++++--------------
 test/unit/middleware/confidenceScore.js       |  49 +++-
 .../middleware/confidenceScoreFallback.js     |  96 ++++++-
 3 files changed, 183 insertions(+), 209 deletions(-)

diff --git a/middleware/confidenceScoreFallback.js b/middleware/confidenceScoreFallback.js
index cead1a35..8593ed60 100644
--- a/middleware/confidenceScoreFallback.js
+++ b/middleware/confidenceScoreFallback.js
@@ -10,6 +10,7 @@
  */
 
 var check = require('check-types');
+var logger = require('pelias-logger').get('api-confidence');
 
 function setup() {
   return computeScores;
@@ -39,217 +40,79 @@ function computeScores(req, res, next) {
  * @returns {object}
  */
 function computeConfidenceScore(req, hit) {
-  var dealBreakers = checkForDealBreakers(req, hit);
-  if (dealBreakers) {
-    hit.confidence = 0.5;
+
+  // if parsed text doesn't exist, which it never should, just assign a low confidence and move on
+  if (!req.clean.hasOwnProperty('parsed_text')) {
+    hit.confidence = 0.1;
+    hit.match_type = 'unknown';
     return hit;
   }
 
-  var checkCount = 3;
-  hit.confidence = 0;
-
-  if (RELATIVE_SCORES) {
-    checkCount += 2;
-    hit.confidence += checkDistanceFromMean(hit._score, mean, stdev);
-    hit.confidence += computeZScore(hit._score, mean, stdev);
-  }
-  hit.confidence += checkName(req.clean.text, req.clean.parsed_text, hit);
-  hit.confidence += checkQueryType(req.clean.parsed_text, hit);
-  hit.confidence += checkAddress(req.clean.parsed_text, hit);
+  // start with a confidence level of 1 because we trust ES queries to be accurate
+  hit.confidence = 1.0;
 
-  // TODO: look at categories and location
+  // in the case of fallback there might be deductions
+  hit.confidence *= checkFallbackLevel(req, hit);
 
-  hit.confidence /= checkCount;
+  // truncate the precision
   hit.confidence = Number((hit.confidence).toFixed(3));
 
   return hit;
 }
 
-/*
- * Check for clearly mismatching properties in a result
- * zip code and state (region) are currently checked if present
- *
- * @param {object|undefined} text
- * @param {object} hit
- * @returns {bool}
- */
-function checkForDealBreakers(req, hit) {
-  if (check.undefined(req.clean.parsed_text)) {
-    return false;
-  }
-
-  if (check.assigned(req.clean.parsed_text.state) && hit.parent.region_a && req.clean.parsed_text.state !== hit.parent.region_a[0]) {
-    logger.debug('[confidence][deal-breaker]: state !== region_a');
-    return true;
+function checkFallbackLevel(req, hit) {
+  if (checkFallbackOccurred(req, hit)) {
+    hit.match_type = 'fallback';
+
+    // if we know a fallback occurred, deduct points based on layer granularity
+    switch (hit.layer) {
+      case 'venue':
+      case 'address':
+        logger.warn('Fallback scenarios should not result in address or venue records!', req.clean.parsed_text);
+        return 0.8;
+      case 'street':
+        return 0.8;
+      case 'locality':
+      case 'borough':
+      case 'neighbourhood':
+        return 0.6;
+      case 'macrocounty':
+      case 'county':
+      case 'localadmin':
+        return 0.4;
+      case 'region':
+        return 0.3;
+      case 'country':
+      case 'dependency':
+      case 'macroregion':
+        return 0.1;
+      default:
+        return 0.1;
+    }
   }
 
-  if (check.assigned(req.clean.parsed_text.postalcode) && check.assigned(hit.address_parts) &&
-      req.clean.parsed_text.postalcode !== hit.address_parts.zip) {
-    return true;
-  }
+  hit.match_type = 'exact';
+  return 1.0;
 }
 
-/**
- * Check how statistically significant the score of this result is
- * given mean and standard deviation
- *
- * @param {number} score
- * @param {number} mean
- * @param {number} stdev
- * @returns {number}
- */
-function checkDistanceFromMean(score, mean, stdev) {
-  return (score - mean) > stdev ? 1 : 0;
-}
+function checkFallbackOccurred(req, hit) {
+  // at this time we only do this for address queries, so keep this simple
+  // TODO: add other layer checks once we start handling disambiguation
 
-/**
- * Compare text string or name component of parsed_text against
- * default name in result
- *
- * @param {string} text
- * @param {object|undefined} parsed_text
- * @param {object} hit
- * @returns {number}
- */
-function checkName(text, parsed_text, hit) {
-  // parsed_text name should take precedence if available since it's the cleaner name property
-  if (check.assigned(parsed_text) && check.assigned(parsed_text.name) &&
-    hit.name.default.toLowerCase() === parsed_text.name.toLowerCase()) {
-    return 1;
-  }
-
-  // if no parsed_text check the text value as provided against result's default name
-  if (hit.name.default.toLowerCase() === text.toLowerCase()) {
-    return 1;
-  }
-
-  // if no matches detected, don't judge too harshly since it was a longshot anyway
-  return 0.7;
+  return (requestedAddress(req) && hit.layer !== 'address') ||
+         (requestedStreet(req) && hit.layer !== 'street');
 }
 
-/**
- * text being set indicates the query was for an address
- * check if house number was specified and found in result
- *
- * @param {object|undefined} text
- * @param {object} hit
- * @returns {number}
- */
-function checkQueryType(text, hit) {
-  if (check.assigned(text) && check.assigned(text.number) &&
-      (check.undefined(hit.address_parts) ||
-      (check.assigned(hit.address_parts) && check.undefined(hit.address_parts.number)))) {
-    return 0;
-  }
-  return 1;
+function requestedAddress(req) {
+  // house number and street name were specified
+  return req.clean.parsed_text.hasOwnProperty('number') &&
+         req.clean.parsed_text.hasOwnProperty('street');
 }
 
-/**
- * Determine the quality of the property match
- *
- * @param {string|number|undefined|null} textProp
- * @param {string|number|undefined|null} hitProp
- * @param {boolean} expectEnriched
- * @returns {number}
- */
-function propMatch(textProp, hitProp, expectEnriched) {
-
-  // both missing, but expect to have enriched value in result => BAD
-  if (check.undefined(textProp) && check.undefined(hitProp) && check.assigned(expectEnriched)) { return 0; }
-
-  // both missing, and no enrichment expected => GOOD
-  if (check.undefined(textProp) && check.undefined(hitProp)) { return 1; }
-
-  // text has it, result doesn't => BAD
-  if (check.assigned(textProp) && check.undefined(hitProp)) { return 0; }
-
-  // text missing, result has it, and enrichment is expected => GOOD
-  if (check.undefined(textProp) && check.assigned(hitProp) && check.assigned(expectEnriched)) { return 1; }
-
-  // text missing, result has it, enrichment not desired => 50/50
-  if (check.undefined(textProp) && check.assigned(hitProp)) { return 0.5; }
-
-  // both present, values match => GREAT
-  if (check.assigned(textProp) && check.assigned(hitProp) &&
-      textProp.toString().toLowerCase() === hitProp.toString().toLowerCase()) { return 1; }
-
-  // ¯\_(ツ)_/¯
-  return 0.7;
-}
-
-/**
- * Check various parts of the parsed text address
- * against the results
- *
- * @param {object} text
- * @param {string|number} [text.number]
- * @param {string} [text.street]
- * @param {string} [text.postalcode]
- * @param {string} [text.state]
- * @param {string} [text.country]
- * @param {object} hit
- * @param {object} [hit.address_parts]
- * @param {string|number} [hit.address_parts.number]
- * @param {string} [hit.address_parts.street]
- * @param {string|number} [hit.address_parts.zip]
- * @param {Array} [hit.parent.region_a]
- * @param {Array} [hit.parent.country_a]
- * @returns {number}
- */
-function checkAddress(text, hit) {
-  var checkCount = 5;
-  var res = 0;
-
-  if (check.assigned(text) && check.assigned(text.number) && check.assigned(text.street)) {
-    res += propMatch(text.number, (hit.address_parts ? hit.address_parts.number : null), false);
-    res += propMatch(text.street, (hit.address_parts ? hit.address_parts.street : null), false);
-    res += propMatch(text.postalcode, (hit.address_parts ? hit.address_parts.zip: null), true);
-    res += propMatch(text.state, (hit.parent.region_a ? hit.parent.region_a[0] : null), true);
-    res += propMatch(text.country, (hit.parent.country_a ? hit.parent.country_a[0] :null), true);
-
-    res /= checkCount;
-  }
-  else {
-    res = 1;
-  }
-
-  return res;
-}
-
-/**
- * z-scores have an effective range of -3.00 to +3.00.
- * An average z-score is ZERO.
- * A negative z-score indicates that the item/element is below
- * average and a positive z-score means that the item/element
- * in above average. When teachers say they are going to "curve"
- * the test, they do this by computing z-scores for the students' test scores.
- *
- * @param {number} score
- * @param {number} mean
- * @param {number} stdev
- * @returns {number}
- */
-function computeZScore(score, mean, stdev) {
-  if (stdev < 0.01) {
-    return 0;
-  }
-  // because the effective range of z-scores is -3.00 to +3.00
-  // add 10 to ensure a positive value, and then divide by 10+3+3
-  // to further normalize to %-like result
-  return (((score - mean) / (stdev)) + 10) / 16;
+function requestedStreet(req) {
+  // only street name was specified
+  return !req.clean.parsed_text.hasOwnProperty('number') &&
+          req.clean.parsed_text.hasOwnProperty('street');
 }
 
-/**
- * Computes standard deviation given an array of values
- *
- * @param {Array} scores
- * @returns {number}
- */
-function computeStandardDeviation(scores) {
-  var stdev = stats.stdev(scores);
-  // if stdev is low, just consider it 0
-  return (stdev < 0.01) ? 0 : stdev;
-}
-
-
 module.exports = setup;
diff --git a/test/unit/middleware/confidenceScore.js b/test/unit/middleware/confidenceScore.js
index 7d6ba87d..a0b4de6f 100644
--- a/test/unit/middleware/confidenceScore.js
+++ b/test/unit/middleware/confidenceScore.js
@@ -46,7 +46,8 @@ module.exports.tests.confidenceScore = function(test, common) {
         }
       }],
       meta: {
-        scores: [10]
+        scores: [10],
+        query_type: 'original'
       }
     };
 
@@ -86,7 +87,10 @@ module.exports.tests.confidenceScore = function(test, common) {
           county: ['city2']
         }
       }],
-      meta: {scores: [10]}
+      meta: {
+        scores: [10],
+        query_type: 'original'
+      }
     };
 
     confidenceScore(req, res, function() {});
@@ -119,13 +123,52 @@ module.exports.tests.confidenceScore = function(test, common) {
           county: ['city1']
         }
       }],
-      meta: {scores: [10]}
+      meta: {
+        scores: [10],
+        query_type: 'original'
+      }
     };
 
     confidenceScore(req, res, function() {});
     t.equal(res.data[0].confidence, 0.28, 'score was set');
     t.end();
   });
+
+  test('should only work for original query_type', function(t) {
+    var req = {
+      clean: {
+        text: '123 Main St, City, NM',
+        parsed_text: {
+          number: 123,
+          street: 'Main St',
+          state: 'NM'
+        }
+      }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1'],
+          region: undefined,
+          region_a: undefined,
+          county: ['city1']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'fallback'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.false(res.data[0].hasOwnProperty('confidence'), 'score was not set');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {
diff --git a/test/unit/middleware/confidenceScoreFallback.js b/test/unit/middleware/confidenceScoreFallback.js
index a0b4de6f..5fcd03de 100644
--- a/test/unit/middleware/confidenceScoreFallback.js
+++ b/test/unit/middleware/confidenceScoreFallback.js
@@ -1,4 +1,4 @@
-var confidenceScore = require('../../../middleware/confidenceScore')();
+var confidenceScore = require('../../../middleware/confidenceScoreFallback')();
 
 module.exports.tests = {};
 
@@ -89,16 +89,16 @@ module.exports.tests.confidenceScore = function(test, common) {
       }],
       meta: {
         scores: [10],
-        query_type: 'original'
+        query_type: 'fallback'
       }
     };
 
     confidenceScore(req, res, function() {});
-    t.equal(res.data[0].confidence, 0.6, 'score was set');
+    t.equal(res.data[0].confidence, 0.1, 'score was set');
     t.end();
   });
 
-  test('undefined region fields should be handled gracefully', function(t) {
+  test('no fallback addresses should have max score', function(t) {
     var req = {
       clean: {
         text: '123 Main St, City, NM',
@@ -114,32 +114,31 @@ module.exports.tests.confidenceScore = function(test, common) {
         _score: 10,
         found: true,
         value: 1,
+        layer: 'address',
         center_point: { lat: 100.1, lon: -50.5 },
         name: { default: 'test name1' },
         parent: {
           country: ['country1'],
-          region: undefined,
-          region_a: undefined,
+          region: ['region1'],
           county: ['city1']
         }
       }],
       meta: {
         scores: [10],
-        query_type: 'original'
+        query_type: 'fallback'
       }
     };
 
     confidenceScore(req, res, function() {});
-    t.equal(res.data[0].confidence, 0.28, 'score was set');
+    t.equal(res.data[0].confidence, 1.0, 'max score was set');
     t.end();
   });
 
-  test('should only work for original query_type', function(t) {
+  test('no fallback street query should have max score', function(t) {
     var req = {
       clean: {
-        text: '123 Main St, City, NM',
+        text: 'Main St, City, NM',
         parsed_text: {
-          number: 123,
           street: 'Main St',
           state: 'NM'
         }
@@ -150,12 +149,12 @@ module.exports.tests.confidenceScore = function(test, common) {
         _score: 10,
         found: true,
         value: 1,
+        layer: 'street',
         center_point: { lat: 100.1, lon: -50.5 },
         name: { default: 'test name1' },
         parent: {
           country: ['country1'],
-          region: undefined,
-          region_a: undefined,
+          region: ['region1'],
           county: ['city1']
         }
       }],
@@ -166,7 +165,76 @@ module.exports.tests.confidenceScore = function(test, common) {
     };
 
     confidenceScore(req, res, function() {});
-    t.false(res.data[0].hasOwnProperty('confidence'), 'score was not set');
+    t.equal(res.data[0].confidence, 1.0, 'max score was set');
+    t.end();
+  });
+
+  test('fallback to locality should have score deduction', function(t) {
+    var req = {
+      clean: {
+        text: '123 Main St, City, NM',
+        parsed_text: {
+          number: 123,
+          street: 'Main St',
+          state: 'NM'
+        }
+      }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        layer: 'locality',
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'fallback'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.equal(res.data[0].confidence, 0.6, 'score was set');
+    t.end();
+  });
+
+  test('fallback to country should have score deduction', function(t) {
+    var req = {
+      clean: {
+        text: '123 Main St, City, NM, USA',
+        parsed_text: {
+          number: 123,
+          street: 'Main St',
+          state: 'NM',
+          country: 'USA'
+        }
+      }
+    };
+    var res = {
+      data: [{
+        _score: 10,
+        found: true,
+        value: 1,
+        layer: 'country',
+        center_point: { lat: 100.1, lon: -50.5 },
+        name: { default: 'test name1' },
+        parent: {
+          country: ['country1']
+        }
+      }],
+      meta: {
+        scores: [10],
+        query_type: 'fallback'
+      }
+    };
+
+    confidenceScore(req, res, function() {});
+    t.equal(res.data[0].confidence, 0.1, 'score was set');
     t.end();
   });
 };

From 3888ae03851a411e2183705423dd2ee253765fc7 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 14:02:14 -0400
Subject: [PATCH 72/82] change query module interfaces to allow for additional
 meta data

---
 controller/search.js               | 11 +++++++++--
 query/autocomplete.js              |  5 ++++-
 query/reverse.js                   |  5 ++++-
 query/search.js                    |  6 +++++-
 query/search_original.js           |  6 +++++-
 test/unit/mock/search_query.js     | 10 ++++------
 test/unit/query/autocomplete.js    | 19 ++++++++++++++++---
 test/unit/query/reverse.js         | 18 ++++++++++++++++--
 test/unit/query/search.js          | 18 ++++++++++++++++--
 test/unit/query/search_original.js | 18 ++++++++++++++++--
 10 files changed, 95 insertions(+), 21 deletions(-)

diff --git a/controller/search.js b/controller/search.js
index 30a6cd1c..5294eac9 100644
--- a/controller/search.js
+++ b/controller/search.js
@@ -10,6 +10,11 @@ function setup( config, backend, query ){
   backend = backend || require('../src/backend');
   query = query || require('../query/search');
 
+  // verify that we don't let an old style query object slip through the cracks here
+  if (typeof query !== 'object' || !query.hasOwnProperty('query_type')) {
+    throw new Error('Search queries must return an object with query and query_type');
+  }
+
   function controller( req, res, next ){
     // do not run controller when a request
     // validation error has occurred.
@@ -31,7 +36,7 @@ function setup( config, backend, query ){
     // log clean parameters for stats
     logger.info('[req]', 'endpoint=' + req.path, cleanOutput);
 
-    var query_body = query(req.clean);
+    var query_body = query.query(req.clean);
 
     // if there's no query to call ES with, skip the service
     if (_.isUndefined(query_body)) {
@@ -61,7 +66,9 @@ function setup( config, backend, query ){
       // set response data
       else {
         res.data = docs;
-        res.meta = meta;
+        res.meta = meta || {};
+        // store the query_type for subsequent middleware
+        res.meta.query_type = query.query_type;
       }
       logger.debug('[ES response]', docs);
       next();
diff --git a/query/autocomplete.js b/query/autocomplete.js
index 5f0826c0..44d3ca21 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -117,4 +117,7 @@ function generateQuery( clean ){
   return query.render( vs );
 }
 
-module.exports = generateQuery;
+module.exports = {
+  query: generateQuery,
+  query_type: 'autocomplete'
+};
diff --git a/query/reverse.js b/query/reverse.js
index 930cba49..e8f8612e 100644
--- a/query/reverse.js
+++ b/query/reverse.js
@@ -74,4 +74,7 @@ function generateQuery( clean ){
   return query.render( vs );
 }
 
-module.exports = generateQuery;
+module.exports = {
+  query: generateQuery,
+  query_type: 'reverse'
+};
diff --git a/query/search.js b/query/search.js
index 33847da5..300a26dc 100644
--- a/query/search.js
+++ b/query/search.js
@@ -134,4 +134,8 @@ function hasStreet(vs) {
   return vs.isset('input:street');
 }
 
-module.exports = generateQuery;
+module.exports = {
+  query: generateQuery,
+  // this could later be set to disambiguation when appropriate
+  query_type: 'fallback'
+};
diff --git a/query/search_original.js b/query/search_original.js
index 4c923246..cc9e5180 100644
--- a/query/search_original.js
+++ b/query/search_original.js
@@ -128,4 +128,8 @@ function generateQuery( clean ){
   return query.render( vs );
 }
 
-module.exports = generateQuery;
+
+module.exports = {
+  query: generateQuery,
+  query_type: 'original'
+};
diff --git a/test/unit/mock/search_query.js b/test/unit/mock/search_query.js
index 2a5f21fd..9bd43cba 100644
--- a/test/unit/mock/search_query.js
+++ b/test/unit/mock/search_query.js
@@ -1,10 +1,8 @@
-
-function setup(){
-  return query;
-}
-
 function query( clean ){
   return clean;
 }
 
-module.exports = setup;
\ No newline at end of file
+module.exports = {
+  query: query,
+  query_type: 'mock'
+};
\ No newline at end of file
diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js
index 437142f3..09ce34ad 100644
--- a/test/unit/query/autocomplete.js
+++ b/test/unit/query/autocomplete.js
@@ -1,11 +1,24 @@
-
-var generate = require('../../../query/autocomplete');
+var query = require('../../../query/autocomplete');
+var generate = query.query;
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof generate, 'function', 'valid function');
+    t.equal(typeof query, 'object', 'valid query object');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.true(query.hasOwnProperty('query'), 'query is valid function');
+    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query, 'function', 'valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query_type, 'string', 'valid query_type');
     t.end();
   });
 };
diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js
index 03985fbd..398eb731 100644
--- a/test/unit/query/reverse.js
+++ b/test/unit/query/reverse.js
@@ -1,10 +1,24 @@
-var generate = require('../../../query/reverse');
+var query = require('../../../query/reverse');
+var generate = query.query;
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof generate, 'function', 'valid function');
+    t.equal(typeof query, 'object', 'valid query object');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.true(query.hasOwnProperty('query'), 'query is valid function');
+    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query, 'function', 'valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query_type, 'string', 'valid query_type');
     t.end();
   });
 };
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index c758a23e..b4a11972 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -1,11 +1,25 @@
-var generate = require('../../../query/search');
+var query = require('../../../query/search');
+var generate = query.query;
 var fs = require('fs');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof generate, 'function', 'valid function');
+    t.equal(typeof query, 'object', 'valid query object');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.true(query.hasOwnProperty('query'), 'query is valid function');
+    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query, 'function', 'valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query_type, 'string', 'valid query_type');
     t.end();
   });
 };
diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js
index b66bbca9..159200b1 100644
--- a/test/unit/query/search_original.js
+++ b/test/unit/query/search_original.js
@@ -1,10 +1,24 @@
-var generate = require('../../../query/search_original');
+var query = require('../../../query/search_original');
+var generate = query.query;
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof generate, 'function', 'valid function');
+    t.equal(typeof query, 'object', 'valid query object');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.true(query.hasOwnProperty('query'), 'query is valid function');
+    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query, 'function', 'valid function');
+    t.end();
+  });
+  test('valid interface', function(t) {
+    t.equal(typeof query.query_type, 'string', 'valid query_type');
     t.end();
   });
 };

From 3cd342f2e72d1edb7f1ca16293f9ab51c6fe1ae1 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 14:03:47 -0400
Subject: [PATCH 73/82] add new confidence score middleware to routes

---
 routes/v1.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/routes/v1.js b/routes/v1.js
index 4890a728..44921e0e 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -37,6 +37,7 @@ var postProc = {
   trimByGranularity: require('../middleware/trimByGranularity'),
   distances: require('../middleware/distance'),
   confidenceScores: require('../middleware/confidenceScore'),
+  confidenceScoresFallback: require('../middleware/confidenceScoreFallback'),
   confidenceScoresReverse: require('../middleware/confidenceScoreReverse'),
   dedupe: require('../middleware/dedupe'),
   localNamingConventions: require('../middleware/localNamingConventions'),
@@ -78,6 +79,7 @@ function addRoutes(app, peliasConfig) {
       postProc.trimByGranularity(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
+      postProc.confidenceScoresFallback(),
       postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),

From d5ff072e9e0f2c4e38dbd6fb2e627dfc25d1543d Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 14:04:43 -0400
Subject: [PATCH 74/82] update tests

---
 test/unit/controller/search.js | 20 +++++++++++---------
 test/unit/run.js               |  1 +
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 408be007..e501518f 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -1,6 +1,6 @@
 var setup = require('../../../controller/search'),
     mockBackend = require('../mock/backend'),
-    mockQuery = require('../mock/query');
+    mockQuery = require('../mock/search_query');
 var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
@@ -47,7 +47,8 @@ module.exports.tests.functional_success = function(test, common) {
   }];
 
   var expectedMeta = {
-    scores: [10, 20]
+    scores: [10, 20],
+    query_type: 'mock'
   };
 
   var expectedData = [
@@ -89,7 +90,7 @@ module.exports.tests.functional_success = function(test, common) {
         searchType: 'dfs_query_then_fetch'
       }, 'correct backend command');
     });
-    var controller = setup(fakeDefaultConfig, backend, mockQuery());
+    var controller = setup(fakeDefaultConfig, backend, mockQuery);
     var res = {
       status: function (code) {
         t.equal(code, 200, 'status set');
@@ -125,7 +126,7 @@ module.exports.tests.functional_success = function(test, common) {
         searchType: 'dfs_query_then_fetch'
       }, 'correct backend command');
     });
-    var controller = setup(fakeCustomizedConfig, backend, mockQuery());
+    var controller = setup(fakeCustomizedConfig, backend, mockQuery);
     var res = {
       status: function (code) {
         t.equal(code, 200, 'status set');
@@ -147,7 +148,7 @@ module.exports.tests.functional_failure = function(test, common) {
     var backend = mockBackend( 'client/search/fail/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'a backend error occurred');
@@ -162,7 +163,7 @@ module.exports.tests.timeout = function(test, common) {
     var backend = mockBackend( 'client/search/timeout/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'Request Timeout after 5000ms');
@@ -177,7 +178,7 @@ module.exports.tests.existing_results = function(test, common) {
     var backend = function() {
       throw new Error('backend should not have been called');
     };
-    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery );
 
     var req = { };
     // the existence of `data` means that there are already results so
@@ -197,8 +198,9 @@ module.exports.tests.existing_results = function(test, common) {
 module.exports.tests.undefined_query = function(test, common) {
   test('query returning undefined should not call service', function(t) {
     // a function that returns undefined
-    var query = function() {
-      return;
+    var query = {
+      query: function () { return; },
+      query_type: 'empty'
     };
 
     var search_service_was_called = false;
diff --git a/test/unit/run.js b/test/unit/run.js
index af155991..35613274 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -26,6 +26,7 @@ var tests = [
   require('./helper/sizeCalculator'),
   require('./middleware/access_log'),
   require('./middleware/confidenceScore'),
+  require('./middleware/confidenceScoreFallback'),
   require('./middleware/confidenceScoreReverse'),
   require('./middleware/distance'),
   require('./middleware/localNamingConventions'),

From 21fcd912c28886e9c3f059f94feff69ee07584e7 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 14:05:25 -0400
Subject: [PATCH 75/82] add match_type to the list of result properties

---
 helper/geojsonify_place_details.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js
index 297ca3bd..6726e60a 100644
--- a/helper/geojsonify_place_details.js
+++ b/helper/geojsonify_place_details.js
@@ -8,6 +8,7 @@ var DETAILS_PROPS = [
   { name: 'street',            type: 'string' },
   { name: 'postalcode',        type: 'string' },
   { name: 'confidence',        type: 'default' },
+  { name: 'match_type',        type: 'string' },
   { name: 'distance',          type: 'default' },
   { name: 'country',           type: 'string' },
   { name: 'country_gid',       type: 'string' },

From cbf2d0ace5832f6e8d5686483278819b789dbad4 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 15:26:59 -0400
Subject: [PATCH 76/82] fix broken ciao tests

---
 middleware/geocodeJSON.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/middleware/geocodeJSON.js b/middleware/geocodeJSON.js
index f4ee8c20..3b5170dd 100644
--- a/middleware/geocodeJSON.js
+++ b/middleware/geocodeJSON.js
@@ -1,6 +1,7 @@
 var url = require('url');
 var extend = require('extend');
 var geojsonify = require('../helper/geojsonify');
+var _ = require('lodash');
 
 /**
  * Returns a middleware function that converts elasticsearch
@@ -79,7 +80,8 @@ function convertToGeocodeJSON(req, res, next, opts) {
 
 function addMessages(req, msgType, geocoding) {
   if (req.hasOwnProperty(msgType) && req[msgType].length) {
-    geocoding[msgType] = req[msgType];
+    // cleanup arrays to make sure there are no duplicates
+    geocoding[msgType] = _.uniq(req[msgType]);
   }
 }
 

From 4a7dee235a166bca19c46bbaa138f3432e7a02ec Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Wed, 21 Sep 2016 16:02:25 -0400
Subject: [PATCH 77/82] add ciao test for confidence and match_type properties

---
 test/ciao/search/address_parsing.coffee |  5 +-
 test/ciao_test_data.js                  | 80 +++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/test/ciao/search/address_parsing.coffee b/test/ciao/search/address_parsing.coffee
index e39f484e..ab98e83e 100644
--- a/test/ciao/search/address_parsing.coffee
+++ b/test/ciao/search/address_parsing.coffee
@@ -38,4 +38,7 @@ json.geocoding.query.parsed_text['number'].should.eql '30'
 json.geocoding.query.parsed_text['street'].should.eql 'w 26th st'
 json.geocoding.query.parsed_text['state'].should.eql 'NY'
 json.geocoding.query.parsed_text['regions'].should.eql []
-json.geocoding.query.parsed_text['admin_parts'].should.eql "ny"
\ No newline at end of file
+json.geocoding.query.parsed_text['admin_parts'].should.eql "ny"
+
+json.features[0].properties.confidence.should.eql 1
+json.features[0].properties.match_type.should.eql "exact"
\ No newline at end of file
diff --git a/test/ciao_test_data.js b/test/ciao_test_data.js
index da1e9821..55872f67 100644
--- a/test/ciao_test_data.js
+++ b/test/ciao_test_data.js
@@ -48,6 +48,86 @@ types.forEach( function( type, i1 ){
   });
 });
 
+client.index(
+  {
+    index: config.indexName,
+    type: 'address',
+    id: 'way:265038872',
+    body: {
+      'center_point': {
+        'lon': -73.990425,
+        'lat': 40.744131
+      },
+      'parent': {
+        'country': [
+          'United States'
+        ],
+        'neighbourhood_id': [
+          '85869245'
+        ],
+        'country_a': [
+          'USA'
+        ],
+        'locality_a': [
+          null
+        ],
+        'region_id': [
+          '85688543'
+        ],
+        'county': [
+          'New York County'
+        ],
+        'borough_a': [
+          null
+        ],
+        'borough_id': [
+          '421205771'
+        ],
+        'locality': [
+          'New York'
+        ],
+        'borough': [
+          'Manhattan'
+        ],
+        'region_a': [
+          'NY'
+        ],
+        'county_id': [
+          '102081863'
+        ],
+        'locality_id': [
+          '85977539'
+        ],
+        'neighbourhood_a': [
+          null
+        ],
+        'neighbourhood': [
+          'Flatiron District'
+        ],
+        'region': [
+          'New York'
+        ],
+        'country_id': [
+          '85633793'
+        ],
+        'county_a': [
+          null
+        ]
+      },
+      'name': {'default': '30 West 26th Street'},
+      'address_parts': {
+        'zip': '10010',
+        'number': '30',
+        'street': 'West 26th Street'
+      },
+      'alpha3': 'USA',
+      'source': 'openstreetmap',
+      'source_id': 'way:265038872',
+      'layer': 'address'
+    }
+  }
+);
+
 // call refresh so the index merges the changes
 actions.push( function( done ){
   client.indices.refresh( { index: config.indexName }, done);

From 9ff383cc2b4a690fa05a88e70c598bfdc28751f4 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Thu, 22 Sep 2016 11:27:20 -0400
Subject: [PATCH 78/82] change the query module interfaces back to simple
 functions

---
 controller/search.js               | 13 +++-----
 query/autocomplete.js              | 10 +++---
 query/reverse.js                   | 10 +++---
 query/search.js                    | 11 +++----
 query/search_original.js           | 10 +++---
 test/unit/controller/search.js     | 17 ++++------
 test/unit/mock/query.js            |  5 ++-
 test/unit/mock/search_query.js     |  8 -----
 test/unit/query/autocomplete.js    | 48 +++++++++++++--------------
 test/unit/query/reverse.js         | 41 ++++++++++-------------
 test/unit/query/search.js          | 51 ++++++++++++++--------------
 test/unit/query/search_original.js | 53 ++++++++++++++----------------
 12 files changed, 123 insertions(+), 154 deletions(-)
 delete mode 100644 test/unit/mock/search_query.js

diff --git a/controller/search.js b/controller/search.js
index 5294eac9..39183fdf 100644
--- a/controller/search.js
+++ b/controller/search.js
@@ -10,11 +10,6 @@ function setup( config, backend, query ){
   backend = backend || require('../src/backend');
   query = query || require('../query/search');
 
-  // verify that we don't let an old style query object slip through the cracks here
-  if (typeof query !== 'object' || !query.hasOwnProperty('query_type')) {
-    throw new Error('Search queries must return an object with query and query_type');
-  }
-
   function controller( req, res, next ){
     // do not run controller when a request
     // validation error has occurred.
@@ -36,10 +31,10 @@ function setup( config, backend, query ){
     // log clean parameters for stats
     logger.info('[req]', 'endpoint=' + req.path, cleanOutput);
 
-    var query_body = query.query(req.clean);
+    var renderedQuery = query(req.clean);
 
     // if there's no query to call ES with, skip the service
-    if (_.isUndefined(query_body)) {
+    if (_.isUndefined(renderedQuery)) {
       return next();
     }
 
@@ -47,7 +42,7 @@ function setup( config, backend, query ){
     var cmd = {
       index: config.indexName,
       searchType: 'dfs_query_then_fetch',
-      body: query_body
+      body: renderedQuery.body
     };
 
     logger.debug( '[ES req]', cmd );
@@ -68,7 +63,7 @@ function setup( config, backend, query ){
         res.data = docs;
         res.meta = meta || {};
         // store the query_type for subsequent middleware
-        res.meta.query_type = query.query_type;
+        res.meta.query_type = renderedQuery.type;
       }
       logger.debug('[ES response]', docs);
       next();
diff --git a/query/autocomplete.js b/query/autocomplete.js
index 44d3ca21..d0e766f3 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -114,10 +114,10 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  return query.render( vs );
+  return {
+    type: 'autocomplete',
+    body: query.render(vs)
+  };
 }
 
-module.exports = {
-  query: generateQuery,
-  query_type: 'autocomplete'
-};
+module.exports = generateQuery;
\ No newline at end of file
diff --git a/query/reverse.js b/query/reverse.js
index e8f8612e..8cb2fa44 100644
--- a/query/reverse.js
+++ b/query/reverse.js
@@ -71,10 +71,10 @@ function generateQuery( clean ){
     vs.var('input:categories', clean.categories);
   }
 
-  return query.render( vs );
+  return {
+    type: 'reverse',
+    body: query.render(vs)
+  };
 }
 
-module.exports = {
-  query: generateQuery,
-  query_type: 'reverse'
-};
+module.exports = generateQuery;
diff --git a/query/search.js b/query/search.js
index 300a26dc..5f7f3e34 100644
--- a/query/search.js
+++ b/query/search.js
@@ -121,7 +121,10 @@ function generateQuery( clean ){
 
 function getQuery(vs) {
   if (hasStreet(vs)) {
-    return fallbackQuery.render(vs);
+    return {
+      type: 'fallback',
+      body: fallbackQuery.render(vs)
+    };
   }
 
   // returning undefined is a signal to a later step that the addressit-parsed
@@ -134,8 +137,4 @@ function hasStreet(vs) {
   return vs.isset('input:street');
 }
 
-module.exports = {
-  query: generateQuery,
-  // this could later be set to disambiguation when appropriate
-  query_type: 'fallback'
-};
+module.exports = generateQuery;
\ No newline at end of file
diff --git a/query/search_original.js b/query/search_original.js
index cc9e5180..e4a0dc66 100644
--- a/query/search_original.js
+++ b/query/search_original.js
@@ -125,11 +125,11 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  return query.render( vs );
+  return {
+    type: 'original',
+    body: query.render(vs)
+  };
 }
 
 
-module.exports = {
-  query: generateQuery,
-  query_type: 'original'
-};
+module.exports = generateQuery;
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index e501518f..86f07b97 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -1,6 +1,6 @@
 var setup = require('../../../controller/search'),
     mockBackend = require('../mock/backend'),
-    mockQuery = require('../mock/search_query');
+    mockQuery = require('../mock/query');
 var proxyquire =  require('proxyquire').noCallThru();
 
 module.exports.tests = {};
@@ -90,7 +90,7 @@ module.exports.tests.functional_success = function(test, common) {
         searchType: 'dfs_query_then_fetch'
       }, 'correct backend command');
     });
-    var controller = setup(fakeDefaultConfig, backend, mockQuery);
+    var controller = setup(fakeDefaultConfig, backend, mockQuery());
     var res = {
       status: function (code) {
         t.equal(code, 200, 'status set');
@@ -126,7 +126,7 @@ module.exports.tests.functional_success = function(test, common) {
         searchType: 'dfs_query_then_fetch'
       }, 'correct backend command');
     });
-    var controller = setup(fakeCustomizedConfig, backend, mockQuery);
+    var controller = setup(fakeCustomizedConfig, backend, mockQuery());
     var res = {
       status: function (code) {
         t.equal(code, 200, 'status set');
@@ -148,7 +148,7 @@ module.exports.tests.functional_failure = function(test, common) {
     var backend = mockBackend( 'client/search/fail/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( fakeDefaultConfig, backend, mockQuery );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'a backend error occurred');
@@ -163,7 +163,7 @@ module.exports.tests.timeout = function(test, common) {
     var backend = mockBackend( 'client/search/timeout/1', function( cmd ){
       t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias', searchType: 'dfs_query_then_fetch' }, 'correct backend command');
     });
-    var controller = setup( fakeDefaultConfig, backend, mockQuery );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
     var req = { clean: { a: 'b' }, errors: [], warnings: [] };
     var next = function(){
       t.equal(req.errors[0],'Request Timeout after 5000ms');
@@ -178,7 +178,7 @@ module.exports.tests.existing_results = function(test, common) {
     var backend = function() {
       throw new Error('backend should not have been called');
     };
-    var controller = setup( fakeDefaultConfig, backend, mockQuery );
+    var controller = setup( fakeDefaultConfig, backend, mockQuery() );
 
     var req = { };
     // the existence of `data` means that there are already results so
@@ -198,10 +198,7 @@ module.exports.tests.existing_results = function(test, common) {
 module.exports.tests.undefined_query = function(test, common) {
   test('query returning undefined should not call service', function(t) {
     // a function that returns undefined
-    var query = {
-      query: function () { return; },
-      query_type: 'empty'
-    };
+    var query = function () { return; };
 
     var search_service_was_called = false;
 
diff --git a/test/unit/mock/query.js b/test/unit/mock/query.js
index 2a5f21fd..a3209a4d 100644
--- a/test/unit/mock/query.js
+++ b/test/unit/mock/query.js
@@ -4,7 +4,10 @@ function setup(){
 }
 
 function query( clean ){
-  return clean;
+  return {
+    type: 'mock',
+    body: clean
+  };
 }
 
 module.exports = setup;
\ No newline at end of file
diff --git a/test/unit/mock/search_query.js b/test/unit/mock/search_query.js
deleted file mode 100644
index 9bd43cba..00000000
--- a/test/unit/mock/search_query.js
+++ /dev/null
@@ -1,8 +0,0 @@
-function query( clean ){
-  return clean;
-}
-
-module.exports = {
-  query: query,
-  query_type: 'mock'
-};
\ No newline at end of file
diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js
index 09ce34ad..64b6c447 100644
--- a/test/unit/query/autocomplete.js
+++ b/test/unit/query/autocomplete.js
@@ -1,24 +1,10 @@
-var query = require('../../../query/autocomplete');
-var generate = query.query;
+var generate = require('../../../query/autocomplete');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof query, 'object', 'valid query object');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.true(query.hasOwnProperty('query'), 'query is valid function');
-    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query, 'function', 'valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query_type, 'string', 'valid query_type');
+    t.equal(typeof generate, 'function', 'valid function');
     t.end();
   });
 };
@@ -35,7 +21,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_only');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_only');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_only');
     t.end();
   });
 
@@ -50,7 +37,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_multiple_tokens');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_multiple_tokens');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_multiple_tokens');
     t.end();
   });
 
@@ -70,7 +58,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_with_admin');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_with_admin');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_with_admin');
     t.end();
   });
 
@@ -88,7 +77,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_final_token');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_final_token');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_final_token');
     t.end();
   });
 
@@ -105,7 +95,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_focus');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_focus');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_focus');
     t.end();
   });
 
@@ -122,7 +113,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_linguistic_focus_null_island');
 
-    t.deepEqual(compiled, expected, 'autocomplete_linguistic_focus_null_island');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_focus_null_island');
     t.end();
   });
 
@@ -138,7 +130,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_with_source_filtering');
 
-    t.deepEqual(compiled, expected, 'valid autocomplete query with source filtering');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid autocomplete query with source filtering');
     t.end();
   });
 
@@ -154,7 +147,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_with_layer_filtering');
 
-    t.deepEqual(compiled, expected, 'valid autocomplete query with layer filtering');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid autocomplete query with layer filtering');
     t.end();
   });
 
@@ -174,7 +168,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_single_character_street');
 
-    t.deepEqual(compiled, expected, 'autocomplete_single_character_street');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete_single_character_street');
     t.end();
   });
 
@@ -190,7 +185,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/autocomplete_boundary_country');
 
-    t.deepEqual(compiled, expected, 'autocomplete: valid boundary.country query');
+    t.deepEqual(compiled.type, 'autocomplete', 'query type set');
+    t.deepEqual(compiled.body, expected, 'autocomplete: valid boundary.country query');
     t.end();
   });
 };
diff --git a/test/unit/query/reverse.js b/test/unit/query/reverse.js
index 398eb731..24ada46a 100644
--- a/test/unit/query/reverse.js
+++ b/test/unit/query/reverse.js
@@ -1,24 +1,10 @@
-var query = require('../../../query/reverse');
-var generate = query.query;
+var generate = require('../../../query/reverse');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof query, 'object', 'valid query object');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.true(query.hasOwnProperty('query'), 'query is valid function');
-    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query, 'function', 'valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query_type, 'string', 'valid query_type');
+    t.equal(typeof generate, 'function', 'valid function');
     t.end();
   });
 };
@@ -36,7 +22,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_standard');
 
-    t.deepEqual(compiled, expected, 'reverse_standard');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'reverse_standard');
     t.end();
   });
 
@@ -52,7 +39,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_null_island');
 
-    t.deepEqual(compiled, expected, 'reverse_null_island');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'reverse_null_island');
     t.end();
   });
 
@@ -68,7 +56,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = '123km';
 
-    t.deepEqual(compiled.query.bool.filter[0].geo_distance.distance, expected, 'distance set to boundary circle radius');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body.query.bool.filter[0].geo_distance.distance, expected, 'distance set to boundary circle radius');
     t.end();
   });
 
@@ -85,8 +74,9 @@ module.exports.tests.query = function(test, common) {
 
     // this should not equal `point.lat` and `point.lon` as it was explitely specified
     var expected = { lat: clean['boundary.circle.lat'], lon: clean['boundary.circle.lon'] };
-    var centroid = compiled.query.bool.filter[0].geo_distance.center_point;
+    var centroid = compiled.body.query.bool.filter[0].geo_distance.center_point;
 
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
     t.deepEqual(centroid, expected, 'reverse: boundary.circle/lon overrides point.lat/lon');
     t.end();
   });
@@ -101,7 +91,7 @@ module.exports.tests.query = function(test, common) {
       });
 
       var compiled = JSON.parse( JSON.stringify( query ) );
-      t.equal( compiled.size, expected[index], 'valid reverse query for size: '+ size);
+      t.equal( compiled.body.size, expected[index], 'valid reverse query for size: '+ size);
     });
     t.end();
   });
@@ -119,7 +109,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_with_boundary_country');
 
-    t.deepEqual(compiled, expected, 'valid reverse query with boundary.country');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid reverse query with boundary.country');
     t.end();
   });
 
@@ -136,7 +127,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_with_source_filtering');
 
-    t.deepEqual(compiled, expected, 'valid reverse query with source filtering');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid reverse query with source filtering');
     t.end();
   });
 
@@ -153,7 +145,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/reverse_with_layer_filtering');
 
-    t.deepEqual(compiled, expected, 'valid reverse query with source filtering');
+    t.deepEqual(compiled.type, 'reverse', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid reverse query with source filtering');
     t.end();
   });
 };
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index b4a11972..36fbd17d 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -1,25 +1,11 @@
-var query = require('../../../query/search');
-var generate = query.query;
+var generate = require('../../../query/search');
 var fs = require('fs');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof query, 'object', 'valid query object');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.true(query.hasOwnProperty('query'), 'query is valid function');
-    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query, 'function', 'valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query_type, 'string', 'valid query_type');
+    t.equal(typeof generate, 'function', 'valid function');
     t.end();
   });
 };
@@ -45,7 +31,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_bbox');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox');
     t.end();
   });
 
@@ -68,7 +55,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_bbox');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_bbox');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_bbox');
     t.end();
   });
 
@@ -86,7 +74,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_only');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_only');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_only');
     t.end();
   });
 
@@ -105,7 +94,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus');
     t.end();
   });
 
@@ -127,7 +117,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_viewport');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_viewport');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_viewport');
     t.end();
   });
 
@@ -151,7 +142,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
 
-    t.deepEqual(compiled, expected, 'valid search query');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid search query');
     t.end();
   });
 
@@ -170,7 +162,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_null_island');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island');
     t.end();
   });
 
@@ -196,7 +189,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse(JSON.stringify(query));
     var expected = require('../fixture/search_fallback');
 
-    t.deepEqual(compiled, expected, 'fallbackQuery');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'fallbackQuery');
     t.end();
 
   });
@@ -234,7 +228,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_boundary_country');
 
-    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search: valid boundary.country query');
     t.end();
   });
 
@@ -252,7 +247,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_source_filtering');
 
-    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering');
     t.end();
   });
 
@@ -270,7 +266,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_category_filtering');
 
-    t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    t.deepEqual(compiled.type, 'fallback', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid search query with category filtering');
     t.end();
   });
 };
diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js
index 159200b1..5e035679 100644
--- a/test/unit/query/search_original.js
+++ b/test/unit/query/search_original.js
@@ -1,24 +1,10 @@
-var query = require('../../../query/search_original');
-var generate = query.query;
+var generate = require('../../../query/search_original');
 
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
   test('valid interface', function(t) {
-    t.equal(typeof query, 'object', 'valid query object');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.true(query.hasOwnProperty('query'), 'query is valid function');
-    t.true(query.hasOwnProperty('query_type'), 'query is valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query, 'function', 'valid function');
-    t.end();
-  });
-  test('valid interface', function(t) {
-    t.equal(typeof query.query_type, 'string', 'valid query_type');
+    t.equal(typeof generate, 'function', 'valid function');
     t.end();
   });
 };
@@ -38,7 +24,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_bbox_original');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_bbox');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus_bbox');
     t.end();
   });
 
@@ -55,7 +42,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_bbox_original');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_bbox');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_bbox');
     t.end();
   });
 
@@ -68,7 +56,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_only_original');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_only');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_only');
     t.end();
   });
 
@@ -82,7 +71,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_original');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus');
     t.end();
   });
 
@@ -96,7 +86,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_linguistic_focus_null_island_original');
 
-    t.deepEqual(compiled, expected, 'search_linguistic_focus_null_island');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_linguistic_focus_null_island');
     t.end();
   });
 
@@ -117,7 +108,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_full_address_original');
 
-    t.deepEqual(compiled, expected, 'search_full_address');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_full_address');
     t.end();
   });
 
@@ -135,7 +127,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_partial_address_original');
 
-    t.deepEqual(compiled, expected, 'search_partial_address');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_partial_address');
     t.end();
   });
 
@@ -147,13 +140,14 @@ module.exports.tests.query = function(test, common) {
         street: 'water st',
         state: 'NY',
         regions: [ 'manhattan' ]
-      },
+      }
     });
 
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_regions_address_original');
 
-    t.deepEqual(compiled, expected, 'search_regions_address');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search_regions_address');
     t.end();
   });
 
@@ -167,7 +161,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_boundary_country_original');
 
-    t.deepEqual(compiled, expected, 'search: valid boundary.country query');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search: valid boundary.country query');
     t.end();
   });
 
@@ -180,7 +175,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_source_filtering_original');
 
-    t.deepEqual(compiled, expected, 'search: valid search query with source filtering');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'search: valid search query with source filtering');
     t.end();
   });
 
@@ -193,7 +189,8 @@ module.exports.tests.query = function(test, common) {
     var compiled = JSON.parse( JSON.stringify( query ) );
     var expected = require('../fixture/search_with_category_filtering_original');
 
-    t.deepEqual(compiled, expected, 'valid search query with category filtering');
+    t.deepEqual(compiled.type, 'original', 'query type set');
+    t.deepEqual(compiled.body, expected, 'valid search query with category filtering');
     t.end();
   });
 };

From 21d4bb63f33bf00c5f75f9d45594b1571f7dcdc3 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Fri, 16 Sep 2016 16:44:27 -0400
Subject: [PATCH 79/82] feat: add accuracy field

---
 helper/geojsonify_place_details.js |  1 +
 middleware/accuracy.js             | 57 +++++++++++++++++++
 routes/v1.js                       |  6 ++
 test/unit/middleware/accuracy.js   | 90 ++++++++++++++++++++++++++++++
 test/unit/run.js                   |  1 +
 5 files changed, 155 insertions(+)
 create mode 100644 middleware/accuracy.js
 create mode 100644 test/unit/middleware/accuracy.js

diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js
index 6726e60a..b936cd41 100644
--- a/helper/geojsonify_place_details.js
+++ b/helper/geojsonify_place_details.js
@@ -10,6 +10,7 @@ var DETAILS_PROPS = [
   { name: 'confidence',        type: 'default' },
   { name: 'match_type',        type: 'string' },
   { name: 'distance',          type: 'default' },
+  { name: 'accuracy',          type: 'string' },
   { name: 'country',           type: 'string' },
   { name: 'country_gid',       type: 'string' },
   { name: 'country_a',         type: 'string' },
diff --git a/middleware/accuracy.js b/middleware/accuracy.js
new file mode 100644
index 00000000..8c83671f
--- /dev/null
+++ b/middleware/accuracy.js
@@ -0,0 +1,57 @@
+/**
+ *
+ * Accuracy level should be set for each item in the results.
+ * The level can be any of the following:
+ *  - point
+ *  - interpolated
+ *  - centroid
+ */
+
+var check = require('check-types');
+
+var accuracyLevel_point = 'point';
+var accuracyLevel_interpolated = 'interpolated';
+var accuracyLevel_centroid = 'centroid';
+
+
+function setup() {
+  return computeAccuracy;
+}
+
+function computeAccuracy(req, res, next) {
+  // do nothing if no result data set
+  if (check.undefined(res) || check.undefined(res.data)) {
+    return next();
+  }
+
+  // loop through data items and determine accuracy levels
+  res.data = res.data.map(computeAccuracyLevelForResult);
+
+  next();
+}
+
+/**
+ * Determine accuracy level based on the type of result being returned.
+ *
+ * @param {object} hit
+ * @returns {object}
+ */
+function computeAccuracyLevelForResult(hit) {
+
+  // TODO: add a check for interpolated addresses when that feature lands
+
+  switch (hit.layer) {
+    case 'venue':
+    case 'address':
+      hit.accuracy = accuracyLevel_point;
+      break;
+    // this means it's a street or admin area
+    default:
+      hit.accuracy = accuracyLevel_centroid;
+      break;
+  }
+
+  return hit;
+}
+
+module.exports = setup;
diff --git a/routes/v1.js b/routes/v1.js
index 44921e0e..362ed2cd 100644
--- a/routes/v1.js
+++ b/routes/v1.js
@@ -39,6 +39,7 @@ var postProc = {
   confidenceScores: require('../middleware/confidenceScore'),
   confidenceScoresFallback: require('../middleware/confidenceScoreFallback'),
   confidenceScoresReverse: require('../middleware/confidenceScoreReverse'),
+  accuracy: require('../middleware/accuracy'),
   dedupe: require('../middleware/dedupe'),
   localNamingConventions: require('../middleware/localNamingConventions'),
   renamePlacenames: require('../middleware/renamePlacenames'),
@@ -81,6 +82,7 @@ function addRoutes(app, peliasConfig) {
       postProc.confidenceScores(peliasConfig),
       postProc.confidenceScoresFallback(),
       postProc.dedupe(),
+      postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
@@ -94,6 +96,7 @@ function addRoutes(app, peliasConfig) {
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
       postProc.dedupe(),
+      postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
@@ -110,6 +113,7 @@ function addRoutes(app, peliasConfig) {
       //  so it must be calculated first
       postProc.confidenceScoresReverse(),
       postProc.dedupe(),
+      postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
@@ -126,6 +130,7 @@ function addRoutes(app, peliasConfig) {
       //  so it must be calculated first
       postProc.confidenceScoresReverse(),
       postProc.dedupe(),
+      postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
@@ -136,6 +141,7 @@ function addRoutes(app, peliasConfig) {
     place: createRouter([
       sanitisers.place.middleware,
       controllers.place(peliasConfig),
+      postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
diff --git a/test/unit/middleware/accuracy.js b/test/unit/middleware/accuracy.js
new file mode 100644
index 00000000..13c14431
--- /dev/null
+++ b/test/unit/middleware/accuracy.js
@@ -0,0 +1,90 @@
+var accuracy = require('../../../middleware/accuracy')();
+
+module.exports.tests = {};
+
+module.exports.tests.accuracy = function(test, common) {
+
+  test('empty res and req should not throw exception', function(t) {
+    function testIt() {
+      accuracy({}, {}, function() {});
+    }
+
+    t.doesNotThrow(testIt, 'an exception should not have been thrown');
+    t.end();
+  });
+
+  test('res.results without parsed_text should not throw exception', function(t) {
+    var res = {
+      data: [{
+        layer: 'venue'
+      }]
+    };
+
+    accuracy({}, res, function() {
+      t.equal(res.data[0].accuracy, 'point', 'accuracy was set');
+      t.end();
+    });
+  });
+
+  test('venue should have accuracy set to point', function(t) {
+    var res = {
+      data: [{
+        layer: 'venue'
+      }]
+    };
+
+    accuracy({}, res, function() {
+      t.equal(res.data[0].accuracy, 'point', 'accuracy was set');
+      t.end();
+    });
+  });
+
+  test('address should have accuracy set to point', function(t) {
+    var res = {
+      data: [{
+        layer: 'address'
+      }]
+    };
+
+    accuracy({}, res, function() {
+      t.equal(res.data[0].accuracy, 'point', 'accuracy was set');
+      t.end();
+    });
+  });
+
+  test('region should have accuracy set to centroid', function(t) {
+    var res = {
+      data: [{
+        layer: 'region'
+      }]
+    };
+
+    accuracy({}, res, function() {
+      t.equal(res.data[0].accuracy, 'centroid', 'accuracy was set');
+      t.end();
+    });
+  });
+
+  test('street should have accuracy set to centroid', function(t) {
+    var res = {
+      data: [{
+        layer: 'street'
+      }]
+    };
+
+    accuracy({}, res, function() {
+      t.equal(res.data[0].accuracy, 'centroid', 'accuracy was set');
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('[middleware] confidenceScore: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
diff --git a/test/unit/run.js b/test/unit/run.js
index 35613274..be804d97 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -25,6 +25,7 @@ var tests = [
   require('./helper/type_mapping'),
   require('./helper/sizeCalculator'),
   require('./middleware/access_log'),
+  require('./middleware/accuracy'),
   require('./middleware/confidenceScore'),
   require('./middleware/confidenceScoreFallback'),
   require('./middleware/confidenceScoreReverse'),

From 30e42e2b80becd6624cab1be53ba5a990a960746 Mon Sep 17 00:00:00 2001
From: Diana Shkolnikov <dianashk@gmail.com>
Date: Thu, 22 Sep 2016 12:19:07 -0400
Subject: [PATCH 80/82] cleanup variable name and add ciao test

---
 middleware/accuracy.js                  | 10 +++++-----
 test/ciao/search/address_parsing.coffee |  3 ++-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/middleware/accuracy.js b/middleware/accuracy.js
index 8c83671f..519973c3 100644
--- a/middleware/accuracy.js
+++ b/middleware/accuracy.js
@@ -9,9 +9,9 @@
 
 var check = require('check-types');
 
-var accuracyLevel_point = 'point';
-var accuracyLevel_interpolated = 'interpolated';
-var accuracyLevel_centroid = 'centroid';
+var accuracyLevelPoint = 'point';
+var accuracyLevelInterpolated = 'interpolated';
+var accuracyLevelCentroid = 'centroid';
 
 
 function setup() {
@@ -43,11 +43,11 @@ function computeAccuracyLevelForResult(hit) {
   switch (hit.layer) {
     case 'venue':
     case 'address':
-      hit.accuracy = accuracyLevel_point;
+      hit.accuracy = accuracyLevelPoint;
       break;
     // this means it's a street or admin area
     default:
-      hit.accuracy = accuracyLevel_centroid;
+      hit.accuracy = accuracyLevelCentroid;
       break;
   }
 
diff --git a/test/ciao/search/address_parsing.coffee b/test/ciao/search/address_parsing.coffee
index ab98e83e..5cfdbcce 100644
--- a/test/ciao/search/address_parsing.coffee
+++ b/test/ciao/search/address_parsing.coffee
@@ -41,4 +41,5 @@ json.geocoding.query.parsed_text['regions'].should.eql []
 json.geocoding.query.parsed_text['admin_parts'].should.eql "ny"
 
 json.features[0].properties.confidence.should.eql 1
-json.features[0].properties.match_type.should.eql "exact"
\ No newline at end of file
+json.features[0].properties.match_type.should.eql "exact"
+json.features[0].properties.accuracy.should.eql "point"
\ No newline at end of file

From cad2c5f5ad0e091a10f00df7cb337f0b3efed839 Mon Sep 17 00:00:00 2001
From: Stephen Hess <stephen.k.hess@gmail.com>
Date: Thu, 22 Sep 2016 15:55:33 -0400
Subject: [PATCH 81/82] bumped addressit version, updated corresponding tests

---
 package.json                           |  2 +-
 test/unit/sanitiser/_text_addressit.js | 68 ++++++++++++++++++++++++--
 2 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/package.json b/package.json
index cc3ac6df..7ab9285c 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,7 @@
     "node": ">=0.10.26"
   },
   "dependencies": {
-    "addressit": "git://github.com/dianashk/addressit.git#temp",
+    "addressit": "1.4.0",
     "async": "^2.0.0",
     "check-types": "^7.0.0",
     "elasticsearch": "^11.0.0",
diff --git a/test/unit/sanitiser/_text_addressit.js b/test/unit/sanitiser/_text_addressit.js
index c6f19660..91e1665a 100644
--- a/test/unit/sanitiser/_text_addressit.js
+++ b/test/unit/sanitiser/_text_addressit.js
@@ -19,13 +19,67 @@ module.exports.tests.text_parser = function(test, common) {
     t.end();
   });
 
-  var queries = [
-    { name: 'soho', admin_parts: 'new york' },
+  var usQueries = [
+    { name: 'soho', admin_parts: 'new york', state: 'NY' },
+    { name: '123 main', admin_parts: 'new york', state: 'NY' }
+  ];
+
+  usQueries.forEach(function (query) {
+    test('naive parsing ' + query, function(t) {
+      var raw = {
+        text: query.name + ', ' + query.admin_parts
+      };
+      var clean = {};
+
+      var expected_clean = {
+        text: query.name + ', ' + query.admin_parts,
+        parsed_text: {
+          name: query.name,
+          regions: [ query.name ],
+          admin_parts: query.admin_parts,
+          state: query.state
+        }
+      };
+
+      var messages = sanitiser(raw, clean);
+
+      t.deepEqual(messages, { errors: [], warnings: [] } );
+      t.deepEqual(clean, expected_clean);
+      t.end();
+
+    });
+
+    test('naive parsing ' + query + ' without spaces', function(t) {
+      var raw = {
+        text: query.name + ',' + query.admin_parts
+      };
+      var clean = {};
+
+      var expected_clean = {
+        text: query.name + ',' + query.admin_parts,
+        parsed_text: {
+          name: query.name,
+          regions: [ query.name ],
+          admin_parts: query.admin_parts,
+          state: query.state
+        }
+      };
+
+      var messages = sanitiser(raw, clean);
+
+      t.deepEqual(messages, { errors: [], warnings: [] } );
+      t.deepEqual(clean, expected_clean);
+      t.end();
+
+    });
+
+  });
+
+  var nonUSQueries = [
     { name: 'chelsea', admin_parts: 'london' },
-    { name: '123 main', admin_parts: 'new york' }
   ];
 
-  queries.forEach(function (query) {
+  nonUSQueries.forEach(function (query) {
     test('naive parsing ' + query, function(t) {
       var raw = {
         text: query.name + ', ' + query.admin_parts
@@ -158,7 +212,11 @@ module.exports.tests.text_parser = function(test, common) {
     clean.parsed_text = 'this should be removed';
 
     var expected_clean = {
-      text: 'main particle new york'
+      text: 'main particle new york',
+      parsed_text: {
+        regions: [ 'main particle' ],
+        state: 'NY'
+      }
     };
 
     var messages = sanitiser(raw, clean);

From a48533fd1ff1568105c8d2ec8505cbe96605b801 Mon Sep 17 00:00:00 2001
From: greenkeeperio-bot <support@greenkeeper.io>
Date: Mon, 3 Oct 2016 11:19:21 -0400
Subject: [PATCH 82/82] chore(package): update pelias-query to version 8.6.0

https://greenkeeper.io/
---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index a06f4282..af54ec1e 100644
--- a/package.json
+++ b/package.json
@@ -54,7 +54,7 @@
     "pelias-config": "2.1.0",
     "pelias-logger": "0.0.8",
     "pelias-model": "4.2.0",
-    "pelias-query": "8.5.0",
+    "pelias-query": "8.6.0",
     "pelias-text-analyzer": "1.3.0",
     "stats-lite": "2.0.3",
     "through2": "2.0.1"