merge master, fix tests

9 years ago · 014cb06bad
14 changed files with 163 additions and 56 deletions
--- a/middleware/dedupe.js
+++ b/middleware/dedupe.js
@ -92,7 +92,7 @@ function normalizeString(str) {
  if (!str) {
    return '';
  }
-  return _.words(str.toLowerCase()).join(' ');
+  return str.toLowerCase().split(/[ ,-]+/).join(' ');
 }


--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@ -162,11 +162,7 @@ function generateQuery( clean ){
    textParser( clean.parsed_text, vs );
  }

-  var q = query.render( vs );
-
-  console.log( JSON.stringify( q, null, 2 ) );
-
-  return q;
+  return query.render( vs );
 }

 module.exports = generateQuery;
--- a/query/search.js
+++ b/query/search.js
@ -73,8 +73,8 @@ function generateQuery( clean ){
    // calculate the centroid from the viewport box
    vs.set({
      'focus:point:lat': clean['focus.viewport.min_lat'] + ( clean['focus.viewport.max_lat'] - clean['focus.viewport.min_lat'] ) / 2,
-      'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2,
-      'focus:scale': calculateDiagonalDistance(clean) + 'km'
+      'focus:point:lon': clean['focus.viewport.min_lon'] + ( clean['focus.viewport.max_lon'] - clean['focus.viewport.min_lon'] ) / 2
+      //, 'focus:scale': calculateDiagonalDistance(clean) + 'km'
    });
  }

--- a/test/unit/fixture/autocomplete_linguistic_final_token.js
+++ b/test/unit/fixture/autocomplete_linguistic_final_token.js
@ -8,7 +8,7 @@ module.exports = {
            'match': {
              'name.default': {
                'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                'query': 'one',
                'type': 'phrase',
                'operator': 'and'
@ -75,6 +75,6 @@ module.exports = {
    }
  },
  'sort': [ '_score' ],
-  'size': 10,
+  'size': 20,
  'track_scores': true
 };
--- a/test/unit/fixture/autocomplete_linguistic_focus.js
+++ b/test/unit/fixture/autocomplete_linguistic_focus.js
@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');

 module.exports = {
  'query': {
@ -9,7 +8,7 @@ module.exports = {
            'match': {
              'name.default': {
                'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                'query': 'test',
                'type': 'phrase',
                'operator': 'and'
@ -22,7 +21,7 @@ module.exports = {
                'match': {
                  'name.default': {
                    'analyzer': 'peliasPhrase',
-                    'boost': 1,
+                    'boost': 100,
                    'query': 'test',
                    'type': 'phrase',
                    'operator': 'and'
@ -36,15 +35,15 @@ module.exports = {
                      'lat': 29.49136,
                      'lon': -82.50622
                    },
-                    'offset': '1km',
-                    'scale': '50km',
+                    'offset': '10km',
+                    'scale': '250km',
                    'decay': 0.5
                  }
                },
-                'weight': 2
+                'weight': 3
              }],
              'score_mode': 'avg',
-              'boost_mode': 'replace',
+              'boost_mode': 'multiply',
              'filter': {
                'or': [
                  {
@ -130,6 +129,6 @@ module.exports = {
    }
  },
  'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
  'track_scores': true
 };
--- a/test/unit/fixture/autocomplete_linguistic_focus_null_island.js
+++ b/test/unit/fixture/autocomplete_linguistic_focus_null_island.js
@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');

 module.exports = {
  'query': {
@ -9,7 +8,7 @@ module.exports = {
            'match': {
              'name.default': {
                'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                'query': 'test',
                'type': 'phrase',
                'operator': 'and'
@ -22,7 +21,7 @@ module.exports = {
                'match': {
                  'name.default': {
                    'analyzer': 'peliasPhrase',
-                    'boost': 1,
+                    'boost': 100,
                    'query': 'test',
                    'type': 'phrase',
                    'operator': 'and'
@ -36,15 +35,15 @@ module.exports = {
                      'lat': 0,
                      'lon': 0
                    },
-                    'offset': '1km',
-                    'scale': '50km',
+                    'offset': '10km',
+                    'scale': '250km',
                    'decay': 0.5
                  }
                },
-                'weight': 2
+                'weight': 3
              }],
              'score_mode': 'avg',
-              'boost_mode': 'replace',
+              'boost_mode': 'multiply',
              'filter': {
                'or': [
                  {
@ -130,6 +129,6 @@ module.exports = {
    }
  },
  'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
  'track_scores': true
 };
--- a/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js
+++ b/test/unit/fixture/autocomplete_linguistic_multiple_tokens.js
@ -19,7 +19,7 @@ module.exports = {
            'match': {
              'name.default': {
                'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                'query': 'three',
                'type': 'phrase',
                'operator': 'and'
@ -86,6 +86,6 @@ module.exports = {
    }
  },
  'sort': [ '_score' ],
-  'size': 10,
+  'size': 20,
  'track_scores': true
 };
--- a/test/unit/fixture/autocomplete_linguistic_only.js
+++ b/test/unit/fixture/autocomplete_linguistic_only.js
@ -1,4 +1,3 @@
-var vs = require('../../../query/autocomplete_defaults');

 module.exports = {
  'query': {
@ -9,7 +8,7 @@ module.exports = {
            'match': {
              'name.default': {
                'analyzer': 'peliasPhrase',
-                'boost': 1,
+                'boost': 100,
                'query': 'test',
                'type': 'phrase',
                'operator': 'and'
@ -76,6 +75,6 @@ module.exports = {
    }
  },
  'sort': [ '_score' ],
-  'size': vs.size,
+  'size': 20,
  'track_scores': true
 };
--- a/test/unit/fixture/autocomplete_linguistic_with_admin.js
+++ b/test/unit/fixture/autocomplete_linguistic_with_admin.js
@ -22,7 +22,7 @@ module.exports = {
              'match': {
                'admin0': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 4,
+                  'boost': 800,
                  'query': 'three'
                }
              }
@ -31,7 +31,7 @@ module.exports = {
              'match': {
                'admin1': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 3,
+                  'boost': 600,
                  'query': 'three'
                }
              }
@ -40,7 +40,7 @@ module.exports = {
              'match': {
                'admin1_abbr': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 3,
+                  'boost': 600,
                  'query': 'three'
                }
              }
@ -49,7 +49,7 @@ module.exports = {
              'match': {
                'admin2': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 2,
+                  'boost': 400,
                  'query': 'three'
                }
              }
@ -58,7 +58,7 @@ module.exports = {
              'match': {
                'local_admin': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                  'query': 'three'
                }
              }
@ -67,7 +67,7 @@ module.exports = {
              'match': {
                'locality': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                  'query': 'three'
                }
              }
@ -76,7 +76,7 @@ module.exports = {
              'match': {
                'neighborhood': {
                  'analyzer': 'peliasAdmin',
-                  'boost': 1,
+                  'boost': 200,
                  'query': 'three'
                }
              }
@ -146,7 +146,7 @@ module.exports = {
      }
    }
  },
-  'size': 10,
+  'size': 20,
  'track_scores': true,
  'sort': [
    '_score'
--- a/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js
+++ b/test/unit/fixture/dedupe_elasticsearch_nonascii_results.js
@ -0,0 +1,61 @@
+module.exports = [
+  {
+    'id': 'foobar',
+    'gid': 'osm:venue:foobar',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里长城万里长城'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Beijing',
+    'confidence': 0.733
+  },
+  {
+    'id': '185883777',
+    'gid': 'osm:venue:185883777',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里长城'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Beijing',
+    'confidence': 0.733
+  },
+  {
+    'id': '1877602615',
+    'gid': 'osm:venue:1877602615',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里花'
+    },
+    'country_a': 'JPN',
+    'country': 'Japan',
+    'region': 'Tokyo',
+    'county': '豊島区',
+    'locality': 'Tokyo',
+    'neighbourhood': '２丁目',
+    'confidence': 0.646
+  },
+  {
+    'id': '231404818',
+    'gid': 'osm:venue:231404818',
+    'layer': 'venue',
+    'source': 'osm',
+    'name': {
+      'default': '万里加油站'
+    },
+    'address': {
+      'street': 'S308',
+      'postalcode': '312044'
+    },
+    'country_a': 'CHN',
+    'country': 'China',
+    'region': 'Zhejiang',
+    'confidence': 0.646
+  }
+];
--- a/test/unit/fixture/search_linguistic_viewport.js
+++ b/test/unit/fixture/search_linguistic_viewport.js
@ -49,7 +49,7 @@ module.exports = {
                          'lon': -82.50622
                        },
                        'offset': '1km',
-                        'scale': '994km',
+                        'scale': '50km',
                        'decay': 0.5
                      }
                    }
--- a/test/unit/middleware/confidenceScore.js
+++ b/test/unit/middleware/confidenceScore.js
@ -41,6 +41,39 @@ module.exports.tests.confidenceScore = function(test, common) {

  });

+  test('hit without address should not error', function(t) {
+    var req = {
+      clean: {
+        text: 'test name3',
+        parsed_text: {
+          postalcode: 12345
+        }
+      }
+    };
+    var res = {
+      data: [{
+        name: {
+          default: 'foo'
+        }
+      }],
+      meta: {
+        scores: [10]
+      }
+    };
+
+    try {
+      confidenceScore(req, res, function() {});
+      t.pass('no exception');
+    }
+    catch (e) {
+      t.fail('an exception should not have been thrown with no address');
+      console.log(e.stack);
+    }
+    finally {
+      t.end();
+    }
+  });
+

  test('res.results without parsed_text should not throw exception', function(t) {
    var req = {
--- a/test/unit/middleware/dedupe.js
+++ b/test/unit/middleware/dedupe.js
@ -1,4 +1,5 @@
 var data = require('../fixture/dedupe_elasticsearch_results');
+var nonAsciiData = require('../fixture/dedupe_elasticsearch_nonascii_results');
 var dedupe = require('../../../middleware/dedupe')();

 module.exports.tests = {};
@ -22,6 +23,23 @@ module.exports.tests.dedupe = function(test, common) {
    });
  });

+  test('handle non-ascii gracefully', function(t) {
+    var req = {
+      clean: {
+        size: 100
+      }
+    };
+    var res = {
+      data: nonAsciiData
+    };
+
+    var expectedCount = 4;
+    dedupe(req, res, function () {
+      t.equal(res.data.length, expectedCount, 'none were removed');
+      t.end();
+    });
+  });
+
  test('truncate results based on specified size', function(t) {
    var req = {
      clean: {
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@ -90,22 +90,24 @@ module.exports.tests.query = function(test, common) {
    t.end();
  });

-  test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
-    var query = generate({
-      text: 'test', querySize: 10,
-      'focus.viewport.min_lat': 28.49135,
-      'focus.viewport.max_lat': 28.49137,
-      'focus.viewport.min_lon': -87.50622,
-      'focus.viewport.max_lon': -87.50624,
-      layers: ['test']
-    });
-
-    var compiled = JSON.parse( JSON.stringify( query ) );
-    var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
-
-    t.deepEqual(compiled, expected, 'valid search query');
-    t.end();
-  });
+  // viewport scale sizing currently disabled.
+  // ref: https://github.com/pelias/api/pull/388
+  // test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
+  //   var query = generate({
+  //     text: 'test', querySize: 10,
+  //     'focus.viewport.min_lat': 28.49135,
+  //     'focus.viewport.max_lat': 28.49137,
+  //     'focus.viewport.min_lon': -87.50622,
+  //     'focus.viewport.max_lon': -87.50624,
+  //     layers: ['test']
+  //   });
+  //
+  //   var compiled = JSON.parse( JSON.stringify( query ) );
+  //   var expected = require('../fixture/search_linguistic_viewport_min_diagonal');
+  //
+  //   t.deepEqual(compiled, expected, 'valid search query');
+  //   t.end();
+  // });

  test('search search + focus on null island', function(t) {
    var query = generate({