merge master, fix tests

middleware/dedupe.js

@@ -92,7 +92,7 @@ function normalizeString(str) {
   if (!str) {
     return '';
   }
-  return _.words(str.toLowerCase()).join(' ');
+  return str.toLowerCase().split(/[ ,-]+/).join(' ');
 }
 
 

query/autocomplete.js

@@ -162,11 +162,7 @@ function generateQuery( clean ){
     textParser( clean.parsed_text, vs );
   }
 
-  var q = query.render( vs );
+  return query.render( vs );
-
-  console.log( JSON.stringify( q, null, 2 ) );
-
-  return q;
 }
 
 module.exports = generateQuery;

query/search.js

@@ -73,8 +73,8 @@ function generateQuery( clean ){
     // calculate the centroid from the viewport box
     vs.set({
       'focus:point:lat': clean['focus.viewport.min_lat'] + ( clean['focus.viewport.max_lat'] - clean['focus.viewport.min_lat'] ) / 2,
-      'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2,
+      'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2
-      'focus:scale': calculateDiagonalDistance(clean) + 'km'
+      //, 'focus:scale': calculateDiagonalDistance(clean) + 'km'
     });
   }
 

test/unit/fixture/autocomplete_linguistic_final_token.js

@@ -8,7 +8,7 @@ module.exports = {
             'match': {
               'name.default': {
                 'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                 'query': 'one',
                 'type': 'phrase',
                 'operator': 'and'
@@ -75,6 +75,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': 10,
+  'size': 20,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -9,7 +8,7 @@ module.exports = {
             'match': {
               'name.default': {
                 'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
                 'operator': 'and'
@@ -22,7 +21,7 @@ module.exports = {
                 'match': {
                   'name.default': {
                     'analyzer': 'peliasPhrase',
-                    'boost': 1,
+                    'boost': 100,
                     'query': 'test',
                     'type': 'phrase',
                     'operator': 'and'
@@ -36,15 +35,15 @@ module.exports = {
                       'lat': 29.49136,
                       'lon': -82.50622
                     },
-                    'offset': '1km',
+                    'offset': '10km',
-                    'scale': '50km',
+                    'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 2
+                'weight': 3
               }],
               'score_mode': 'avg',
-              'boost_mode': 'replace',
+              'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
@@ -130,6 +129,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -9,7 +8,7 @@ module.exports = {
             'match': {
               'name.default': {
                 'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
                 'operator': 'and'
@@ -22,7 +21,7 @@ module.exports = {
                 'match': {
                   'name.default': {
                     'analyzer': 'peliasPhrase',
-                    'boost': 1,
+                    'boost': 100,
                     'query': 'test',
                     'type': 'phrase',
                     'operator': 'and'
@@ -36,15 +35,15 @@ module.exports = {
                       'lat': 0,
                       'lon': 0
                     },
-                    'offset': '1km',
+                    'offset': '10km',
-                    'scale': '50km',
+                    'scale': '250km',
                     'decay': 0.5
                   }
                 },
-                'weight': 2
+                'weight': 3
               }],
               'score_mode': 'avg',
-              'boost_mode': 'replace',
+              'boost_mode': 'multiply',
               'filter': {
                 'or': [
                   {
@@ -130,6 +129,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_multiple_tokens.js

@@ -19,7 +19,7 @@ module.exports = {
             'match': {
               'name.default': {
                 'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                 'query': 'three',
                 'type': 'phrase',
                 'operator': 'and'
@@ -86,6 +86,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': 10,
+  'size': 20,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_only.js

@@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -9,7 +8,7 @@ module.exports = {
             'match': {
               'name.default': {
                 'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                 'query': 'test',
                 'type': 'phrase',
                 'operator': 'and'
@@ -76,6 +75,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_with_admin.js

@@ -22,7 +22,7 @@ module.exports = {
               'match': {
                 'admin0': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 4,
+                  'boost': 800,
                   'query': 'three'
                 }
               }
@@ -31,7 +31,7 @@ module.exports = {
               'match': {
                 'admin1': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 3,
+                  'boost': 600,
                   'query': 'three'
                 }
               }
@@ -40,7 +40,7 @@ module.exports = {
               'match': {
                 'admin1_abbr': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 3,
+                  'boost': 600,
                   'query': 'three'
                 }
               }
@@ -49,7 +49,7 @@ module.exports = {
               'match': {
                 'admin2': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 2,
+                  'boost': 400,
                   'query': 'three'
                 }
               }
@@ -58,7 +58,7 @@ module.exports = {
               'match': {
                 'local_admin': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                   'query': 'three'
                 }
               }
@@ -67,7 +67,7 @@ module.exports = {
               'match': {
                 'locality': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                   'query': 'three'
                 }
               }
@@ -76,7 +76,7 @@ module.exports = {
               'match': {
                 'neighborhood': {
                   'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                   'query': 'three'
                 }
               }
@@ -146,7 +146,7 @@ module.exports = {
       }
     }
   },
-  'size': 10,
+  'size': 20,
   'track_scores': true,
   'sort': [
     '_score'

test/unit/fixture/dedupe_elasticsearch_nonascii_results.js

@@ -0,0 +1,61 @@
+module.exports = [
+  {
+    'id': 'foobar',
+    'gid': 'osm:venue:foobar',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里长城万里长城'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Beijing',
+    'confidence': 0.733
+  },
+  {
+    'id': '185883777',
+    'gid': 'osm:venue:185883777',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里长城'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Beijing',
+    'confidence': 0.733
+  },
+  {
+    'id': '1877602615',
+    'gid': 'osm:venue:1877602615',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里花'
+    },
+    'country_a': 'JPN',
+    'country': 'Japan',
+    'region': 'Tokyo',
+    'county': '豊島区',
+    'locality': 'Tokyo',
+    'neighbourhood': '２丁目',
+    'confidence': 0.646
+  },
+  {
+    'id': '231404818',
+    'gid': 'osm:venue:231404818',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里加油站'
+    },
+    'address': {
+      'street': 'S308',
+      'postalcode': '312044'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Zhejiang',
+    'confidence': 0.646
+  }
+];

test/unit/fixture/search_linguistic_viewport.js

@@ -49,7 +49,7 @@ module.exports = {
                           'lon': -82.50622
                         },
                         'offset': '1km',
-                        'scale': '994km',
+                        'scale': '50km',
                         'decay': 0.5
                       }
                     }

test/unit/middleware/confidenceScore.js

@@ -41,6 +41,39 @@ module.exports.tests.confidenceScore = function(test, common) {
 
   });
 
+  test('hit without address should not error', function(t) {
+    var req = {
+      clean: {
+        text: 'test name3',
+        parsed_text: {
+          postalcode: 12345
+        }
+      }
+    };
+    var res = {
+      data: [{
+        name: {
+          default: 'foo'
+        }
+      }],
+      meta: {
+        scores: [10]
+      }
+    };
+
+    try {
+      confidenceScore(req, res, function() {});
+      t.pass('no exception');
+    }
+    catch (e) {
+      t.fail('an exception should not have been thrown with no address');
+      console.log(e.stack);
+    }
+    finally {
+      t.end();
+    }
+  });
+
 
   test('res.results without parsed_text should not throw exception', function(t) {
     var req = {

test/unit/middleware/dedupe.js

@@ -1,4 +1,5 @@
 var data = require('../fixture/dedupe_elasticsearch_results');
+var nonAsciiData = require('../fixture/dedupe_elasticsearch_nonascii_results');
 var dedupe = require('../../../middleware/dedupe')();
 
 module.exports.tests = {};
@@ -22,6 +23,23 @@ module.exports.tests.dedupe = function(test, common) {
     });
   });
 
+  test('handle non-ascii gracefully', function(t) {
+    var req = {
+      clean: {
+        size: 100
+      }
+    };
+    var res = {
+      data: nonAsciiData
+    };
+
+    var expectedCount = 4;
+    dedupe(req, res, function () {
+      t.equal(res.data.length, expectedCount, 'none were removed');
+      t.end();
+    });
+  });
+
   test('truncate results based on specified size', function(t) {
     var req = {
       clean: {

test/unit/query/search.js

@@ -90,22 +90,24 @@ module.exports.tests.query = function(test, common) {
     t.end();
   });
 
-  test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
+  // viewport scale sizing currently disabled.
-    var query = generate({
+  // ref: https://github.com/pelias/api/pull/388
-      text: 'test', querySize: 10,
+  // test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
-      'focus.viewport.min_lat': 28.49135,
+  //   var query = generate({
-      'focus.viewport.max_lat': 28.49137,
+  //     text: 'test', querySize: 10,
-      'focus.viewport.min_lon': -87.50622,
+  //     'focus.viewport.min_lat': 28.49135,
-      'focus.viewport.max_lon': -87.50624,
+  //     'focus.viewport.max_lat': 28.49137,
-      layers: ['test']
+  //     'focus.viewport.min_lon': -87.50622,
-    });
+  //     'focus.viewport.max_lon': -87.50624,
-
+  //     layers: ['test']
-    var compiled = JSON.parse( JSON.stringify( query ) );
+  //   });
-    var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
+  //
-
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
-    t.deepEqual(compiled, expected, 'valid search query');
+  //   var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
-    t.end();
+  //
-  });
+  //   t.deepEqual(compiled, expected, 'valid search query');
+  //   t.end();
+  // });
 
   test('search search + focus on null island', function(t) {
     var query = generate({