Merge pull request #985 from pelias/master

Merge master into staging

.circleci/docker.sh

@@ -6,6 +6,11 @@ DATE=`date +%Y-%m-%d`
 DOCKER_REPOSITORY="pelias"
 DOCKER_PROJECT="${DOCKER_REPOSITORY}/${CIRCLE_PROJECT_REPONAME}"
 
+# skip builds on greenkeeper branches
+if [[ -z "${CIRCLE_BRANCH##*greenkeeper*}" ]]; then
+	exit 0
+fi
+
 # the name of the image that represents the "branch", that is an image that will be updated over time with the git branch
 # the production branch is changed to "latest", otherwise the git branch becomes the name of the version
 if [[ "${CIRCLE_BRANCH}" == "production" ]]; then

middleware/confidenceScoreFallback.js

@@ -76,6 +76,8 @@ function checkFallbackLevel(req, hit) {
         return 0.8;
       case 'street':
         return 0.8;
+      case 'postalcode':
+        return 0.8;    
       case 'localadmin':
       case 'locality':
       case 'borough':
@@ -137,38 +139,44 @@ const fallbackRules = [
     expectedLayers: ['street']
   },
   {
-    name: 'neighbourhood',
+    name: 'postalcode',
     notSet: ['query', 'number', 'street'],
+    set: ['postalcode'],
+    expectedLayers: ['postalcode']
+  },
+  {
+    name: 'neighbourhood',
+    notSet: ['query', 'number', 'street', 'postalcode'],
     set: ['neighbourhood'],
     expectedLayers: ['neighbourhood']
   },
   {
     name: 'borough',
-    notSet: ['query', 'number', 'street', 'neighbourhood'],
+    notSet: ['query', 'number', 'street', 'postalcode', 'neighbourhood'],
     set: ['borough'],
     expectedLayers: ['borough']
   },
   {
     name: 'city',
-    notSet: ['query', 'number', 'street', 'neighbourhood', 'borough'],
+    notSet: ['query', 'number', 'street', 'postalcode', 'neighbourhood', 'borough'],
     set: ['city'],
     expectedLayers: ['borough', 'locality', 'localadmin']
   },
   {
     name: 'county',
-    notSet: ['query', 'number', 'street', 'neighbourhood', 'borough', 'city'],
+    notSet: ['query', 'number', 'street', 'postalcode', 'neighbourhood', 'borough', 'city'],
     set: ['county'],
     expectedLayers: ['county']
   },
   {
     name: 'state',
-    notSet: ['query', 'number', 'street', 'neighbourhood', 'borough', 'city', 'county'],
+    notSet: ['query', 'number', 'street', 'postalcode', 'neighbourhood', 'borough', 'city', 'county'],
     set: ['state'],
     expectedLayers: ['region']
   },
   {
     name: 'country',
-    notSet: ['query', 'number', 'street', 'neighbourhood', 'borough', 'city', 'county', 'state'],
+    notSet: ['query', 'number', 'street', 'postalcode', 'neighbourhood', 'borough', 'city', 'county', 'state'],
     set: ['country'],
     expectedLayers: ['country']
   }

middleware/trimByGranularityStructured.js

@@ -24,6 +24,7 @@ const layers = [
   'venue',
   'address',
   'street',
+  'postalcode',
   'neighbourhood',
   ['borough', 'locality'],
   'localadmin',
@@ -46,6 +47,7 @@ const explicit_borough_layers = [
   'venue',
   'address',
   'street',
+  'postalcode',
   'neighbourhood',
   'borough',
   'locality',

package.json

@@ -49,7 +49,7 @@
     "geolib": "^2.0.18",
     "iso-639-3": "^1.0.0",
     "iso3166-1": "^0.3.0",
-    "joi": "^10.5.2",
+    "joi": "^11.0.1",
     "locale": "^0.1.0",
     "lodash": "^4.17.4",
     "markdown": "0.5.0",

query/search.js

@@ -135,7 +135,7 @@ function getQuery(vs) {
 
   logger.info(`[query:search] [search_input_type:${determineQueryType(vs)}]`);
 
-  if (hasStreet(vs) || isPostalCodeOnly(vs)) {
+  if (hasStreet(vs) || isPostalCodeOnly(vs) || isPostalCodeWithCountry(vs)) {
     return {
       type: 'fallback',
       body: fallbackQuery.render(vs)
@@ -182,4 +182,18 @@ function isPostalCodeOnly(vs) {
 
 }
 
+
+function isPostalCodeWithCountry(vs) {
+    var isSet = (layer) => {
+        return vs.isset(`input:${layer}`);
+    };
+    
+    var allowedFields = ['postcode', 'country'];
+    var disallowedFields = ['query', 'category', 'housenumber', 'street', 'locality',
+                          'neighbourhood', 'borough', 'county', 'region'];
+        
+    return allowedFields.every(isSet) &&
+        !disallowedFields.some(isSet);
+}
+
 module.exports = generateQuery;

test/unit/middleware/confidenceScoreFallback.js

@@ -442,6 +442,57 @@ module.exports.tests.confidenceScore = function(test, common) {
     t.equal(res.data[0].match_type, 'fallback', 'fallback match indicated');
     t.end();
   });
+    
+    test('address fallback to postal code should have a score set to 0.8', function(t) {
+        var req = {
+            clean: {
+                text: '123 Main St, City, NM, USA, 1234',
+                parsed_text: {
+                    number: 123,
+                    street: 'Main St',
+                    state: 'NM',
+                    country: 'USA',
+                    postalcode: '1234'
+                }
+            }
+        };
+        var res = {
+            data: [{
+                layer: 'postalcode'
+            }],
+            meta: {
+                query_type: 'fallback'
+            }
+        };
+        confidenceScore(req, res, function() {});
+        t.equal(res.data[0].confidence, 0.8, 'score was set');
+        t.equal(res.data[0].match_type, 'fallback', 'fallback match indicated');
+        t.end();
+    });
+    
+    test('matching address search with postalcode and country should have an exact match with score of 1.0', function(t) {
+        var req = {
+            clean: {
+                text: 'USA, 1234',
+                parsed_text: {
+                    country: 'USA',
+                    postalcode: '1234'
+                }
+            }
+        };
+        var res = {
+            data: [{
+                layer: 'postalcode'
+            }],
+            meta: {
+                query_type: 'fallback'
+            }
+        };
+        confidenceScore(req, res, function() {});
+        t.equal(res.data[0].confidence, 1, 'score was set');
+        t.equal(res.data[0].match_type, 'exact', 'exact match indicated');
+        t.end();
+    });
 
 };
 

test/unit/middleware/trimByGranularityStructured.js

@@ -21,6 +21,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'venue 2', _matched_queries: ['fallback.venue'] },
         { name: 'address 1', _matched_queries: ['fallback.address'] },
         { name: 'street 1', _matched_queries: ['fallback.street'] },
+        { name: 'postalcode 1', _matched_queries: ['fallback.postalcode'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -58,6 +59,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
         { name: 'address 1', _matched_queries: ['fallback.address'] },
         { name: 'address 2', _matched_queries: ['fallback.address'] },
         { name: 'street 1', _matched_queries: ['fallback.street'] },
+        { name: 'postalcode 1', _matched_queries: ['fallback.postalcode'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -94,6 +96,7 @@ module.exports.tests.trimByGranularity = function(test, common) {
       data: [
         { name: 'street 1', _matched_queries: ['fallback.street'] },
         { name: 'street 2', _matched_queries: ['fallback.street'] },
+        { name: 'postalcode 1', _matched_queries: ['fallback.postalcode'] },
         { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
         { name: 'borough 1', _matched_queries: ['fallback.borough'] },
         { name: 'locality 1', _matched_queries: ['fallback.locality'] },
@@ -122,6 +125,49 @@ module.exports.tests.trimByGranularity = function(test, common) {
 
     testIt();
   });
+    
+  test('all records with fallback.* matched_queries name should retain only postalcodes when they are most granular', function(t) {
+    var req = {
+      clean: {
+        parsed_text: {
+          borough: 'borough value'
+        }
+      }
+    };
+
+    var res = {
+      data: [
+        { name: 'postalcode 1', _matched_queries: ['fallback.postalcode'] },
+        { name: 'postalcode 2', _matched_queries: ['fallback.postalcode'] },
+        { name: 'neighbourhood 1', _matched_queries: ['fallback.neighbourhood'] },
+        { name: 'borough 1', _matched_queries: ['fallback.borough'] },
+        { name: 'borough 2', _matched_queries: ['fallback.borough'] },
+        { name: 'locality 1', _matched_queries: ['fallback.locality'] },
+        { name: 'localadmin 1', _matched_queries: ['fallback.localadmin'] },
+        { name: 'county 1', _matched_queries: ['fallback.county'] },
+        { name: 'macrocounty 1', _matched_queries: ['fallback.macrocounty'] },
+        { name: 'region 1', _matched_queries: ['fallback.region'] },
+        { name: 'macroregion 1', _matched_queries: ['fallback.macroregion'] },
+        { name: 'dependency 1', _matched_queries: ['fallback.dependency'] },
+        { name: 'country 1', _matched_queries: ['fallback.country'] },
+        { name: 'unknown', _matched_queries: ['fallback.unknown'] }
+      ]
+    };
+
+    var expected_data = [
+      { name: 'postalcode 1', _matched_queries: ['fallback.postalcode'] },
+      { name: 'postalcode 2', _matched_queries: ['fallback.postalcode'] },
+    ];
+
+    function testIt() {
+      trimByGranularity(req, res, function() {
+        t.deepEquals(res.data, expected_data, 'only postalcode records should be here');
+        t.end();
+      });
+    }
+
+    testIt();
+  });
 
   test('all records with fallback.* matched_queries name should retain only neighbourhoods when they are most granular', function(t) {
     var req = { clean: {} };