added middleware component to trim by result granularity

purpose in comments, cleans up from overly-verbose FallbackQuery

middleware/trimByGranularity.js

@@ -1,34 +1,61 @@
 var _ = require('lodash');
 
+// This middleware component trims the results array by granularity when
+// FallbackQuery was used.  FallbackQuery is used for inputs like
+// `1090 N Charlotte St, Lancaster, PA` where the address may not exist and
+// we must fall back to trying `Lancaster, PA`.  If the address does exist then
+// FallbackQuery will return results for:
+// - address+city+state
+// - city+state
+// - state
+//
+// Because the address matched, we're not interested in city+state or state, so
+// this component removes that aren't the most granular.  
+
 // layers in increasing order of granularity
 var layers = [
-  ['venue'],
-  ['address'],
-  ['neighbourhood'],
-  ['locality', 'localadmin'],
-  ['county', 'macrocounty'],
-  ['region', 'macroregion'],
-  ['country']
+  'venue',
+  'address',
+  'neighbourhood',
+  'borough',
+  'locality',
+  'county',
+  'region',
+  'country'
 ];
 
-function hasRecordsAtLayers(results, layers) {
-  return _.some(results, function(result) {
-    return layers.indexOf(result.layer) !== -1;
+// this helper method returns `true` if every result has a matched_query
+//  starting with `fallback.`
+function isFallbackQuery(results) {
+  return results.every(function(result) {
+    return result.hasOwnProperty('_matched_queries') &&
+            !_.isEmpty(result._matched_queries) &&
+            _.startsWith(result._matched_queries[0], 'fallback.');
+  });
+}
+
+function hasRecordsAtLayers(results, layer) {
+  return results.some(function(result) {
+    return result._matched_queries[0] === 'fallback.' + layer;
   });
 }
 
-function retainRecordsAtLayers(results, layers) {
-  return _.filter(results, function(result) {
-    return layers.indexOf(result.layer) !== -1;
+function retainRecordsAtLayers(results, layer) {
+  return results.filter(function(result) {
+    return result._matched_queries[0] === 'fallback.' + layer;
   });
 }
 
 function setup() {
  return function trim(req, res, next) {
-   if (_.isUndefined(req.clean)) {
+   // don't do anything if there are no results or there are non-fallback.* named queries
+   // there should never be a mixture of fallback.* and non-fallback.* named queries
+   if (_.isUndefined(res.data) || !isFallbackQuery(res.data)) {
      return next();
    }
 
+   // start at the most granular possible layer.  if there are results at a layer
+   // then remove everything not at that layer.
    layers.forEach(function(layer) {
      if (hasRecordsAtLayers(res.data, layer )) {
        res.data = retainRecordsAtLayers(res.data, layer);

test/unit/middleware/trimByGranularity.js

@@ -12,28 +12,26 @@ module.exports.tests.trimByGranularity = function(test, common) {
     t.end();
   });
 
-  test('when venue records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'venue 1', layer: 'venue' },
-        { name: 'venue 2', layer: 'venue' },
-        { name: 'address 1', layer: 'address' },
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' }
+        { name: 'venue 1', _matched_queries: ['fallback.venue'] },
+        { name: 'venue 2', _matched_queries: ['fallback.venue'] },
+        { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'venue 1', layer: 'venue' },
-      { name: 'venue 2', layer: 'venue' },
+      { name: 'venue 1', _matched_queries: ['fallback.venue'] },
+      { name: 'venue 2', _matched_queries: ['fallback.venue'] },
     ];
 
     function testIt() {
@@ -44,30 +42,27 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when address records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only addresses when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'address 1', layer: 'address' },
-        { name: 'address 2', layer: 'address' },
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'address 1', _matched_queries: ['fallback.address'] },
+        { name: 'address 2', _matched_queries: ['fallback.address'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'address 1', layer: 'address' },
-      { name: 'address 2', layer: 'address' }
+      { name: 'address 1', _matched_queries: ['fallback.address'] },
+      { name: 'address 2', _matched_queries: ['fallback.address'] },
     ];
 
     function testIt() {
@@ -78,31 +73,26 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when neighbourhood records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only neighbourhoods when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'neighbourhood 1', layer: 'neighbourhood' },
-        { name: 'neighbourhood 2', layer: 'neighbourhood' },
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'locality 2', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'localadmin 2', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' }
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'neighbourhood 1', layer: 'neighbourhood' },
-      { name: 'neighbourhood 2', layer: 'neighbourhood' }
+      { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+      { name: 'neighbourhood 2', _matched_queries: ['fallback.neighbourhood'] },
     ];
 
     function testIt() {
@@ -113,121 +103,106 @@ module.exports.tests.trimByGranularity = function(test, common) {
     }
 
     testIt();
-
   });
 
-  test('when locality/localadmin records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only localities when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'locality 1', layer: 'locality' },
-        { name: 'locality 2', layer: 'locality' },
-        { name: 'localadmin 1', layer: 'localadmin' },
-        { name: 'localadmin 2', layer: 'localadmin' },
-        { name: 'county 1', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'locality 2', _matched_queries: ['fallback.locality'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'locality 1', layer: 'locality' },
-      { name: 'locality 2', layer: 'locality' },
-      { name: 'localadmin 1', layer: 'localadmin' },
-      { name: 'localadmin 2', layer: 'localadmin' }
+      { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+      { name: 'locality 2', _matched_queries: ['fallback.locality'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only locality/localadmin records should be here');
+        t.deepEquals(res.data, expected_data, 'only locality records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when county/macrocounty records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'county 1', layer: 'county' },
-        { name: 'county 2', layer: 'county' },
-        { name: 'macrocounty 1', layer: 'macrocounty' },
-        { name: 'macrocounty 2', layer: 'macrocounty' },
-        { name: 'region 1', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'county 2', _matched_queries: ['fallback.county'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'county 1', layer: 'county' },
-      { name: 'county 2', layer: 'county' },
-      { name: 'macrocounty 1', layer: 'macrocounty' },
-      { name: 'macrocounty 2', layer: 'macrocounty' },
+      { name: 'county 1', _matched_queries: ['fallback.county'] },
+      { name: 'county 2', _matched_queries: ['fallback.county'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only county/macrocounty records should be here');
+        t.deepEquals(res.data, expected_data, 'only county records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when region/macroregion records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only venues when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'region 1', layer: 'region' },
-        { name: 'region 2', layer: 'region' },
-        { name: 'macroregion 1', layer: 'macroregion' },
-        { name: 'macroregion 2', layer: 'macroregion' },
-        { name: 'country 1', layer: 'country' },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'region 2', _matched_queries: ['fallback.region'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'region 1', layer: 'region' },
-      { name: 'region 2', layer: 'region' },
-      { name: 'macroregion 1', layer: 'macroregion' },
-      { name: 'macroregion 2', layer: 'macroregion' }
+      { name: 'region 1', _matched_queries: ['fallback.region'] },
+      { name: 'region 2', _matched_queries: ['fallback.region'] },
     ];
 
     function testIt() {
       trimByGranularity(req, res, function() {
-        t.deepEquals(res.data, expected_data, 'only region/macroregion records should be here');
+        t.deepEquals(res.data, expected_data, 'only region records should be here');
         t.end();
       });
     }
 
     testIt();
-
   });
 
-  test('when country records are most granular, only they should be retained', function(t) {
+  test('all records with fallback.* matched_queries name should retain only countries when they are most granular', function(t) {
     var req = { clean: {} };
 
     var res = {
       data: [
-        { name: 'country 1', layer: 'country' },
-        { name: 'country 2', layer: 'country' }
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'country 2', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
       ]
     };
 
     var expected_data = [
-      { name: 'country 1', layer: 'country' },
-      { name: 'country 2', layer: 'country' }
+      { name: 'country 1', _matched_queries: ['fallback.country'] },
+      { name: 'country 2', _matched_queries: ['fallback.country'] },
     ];
 
     function testIt() {
@@ -237,6 +212,33 @@ module.exports.tests.trimByGranularity = function(test, common) {
       });
     }
 
+    testIt();
+  });
+
+  test('presence of any non-fallback.* named queries should not trim', function(t) {
+    var req = { clean: {} };
+
+    var res = {
+      data: [
+        { name: 'region', _matched_queries: ['fallback.region'] },
+        { name: 'country', _matched_queries: ['fallback.country'] },
+        { name: 'result with non-named query' }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'region', _matched_queries: ['fallback.region'] },
+      { name: 'country', _matched_queries: ['fallback.country'] },
+      { name: 'result with non-named query' }
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'all should results should have been retained');
+        t.end();
+      });
+    }
+
     testIt();
 
   });