Merge pull request #377 from pelias/master

Merge master into staging

circle.yml

@@ -4,20 +4,14 @@ machine:
   node:
     version: 0.12.2
 
-dependencies:
-  pre:
-    - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json
-
 deployment:
   dev:
     branch: master
     commands:
-      - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install
       - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install
       - cd pelias-deploy && bundle exec rake deploy:api[dev]
-      - cd acceptance-tests && npm test -- -e dev
   prod_build:
-    branch: production
+    branch: staging
     commands:
       - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install
       - cd pelias-deploy && bundle exec rake deploy:api[prod_build]

controller/search.js

@@ -1,4 +1,5 @@
 var service = { search: require('../service/search') };
+var logger = require('pelias-logger').get('api:controller:search');
 
 function setup( backend, query ){
 
@@ -14,6 +15,9 @@ function setup( backend, query ){
       return next();
     }
 
+    // log clean parameters for stats
+    logger.info('[req]', 'endpoint=' + req.path, req.clean);
+
     // backend command
     var cmd = {
       index: 'pelias',

middleware/confidenceScore.js

@@ -100,8 +100,8 @@ function checkForDealBreakers(req, hit) {
     return true;
   }
 
-  if (check.assigned(req.clean.parsed_text.postalcode) && req.clean.parsed_text.postalcode !== hit.zip) {
+  if (check.assigned(req.clean.parsed_text.postalcode) && req.clean.parsed_text.postalcode !== hit.address.zip) {
-    logger.debug('[confidence][deal-breaker]: postalcode !== zip');
+    logger.debug('[confidence][deal-breaker]: postalcode !== zip (' + req.clean.parsed_text.postalcode + ' !== ' + hit.address.zip + ')');
     return true;
   }
 }
@@ -208,7 +208,7 @@ function propMatch(textProp, hitProp, expectEnriched) {
  * @param {object} [hit.address]
  * @param {string|number} [hit.address.number]
  * @param {string} [hit.address.street]
- * @param {string|number} [hit.zip]
+ * @param {string|number} [hit.address.zip]
  * @param {string} [hit.admin1_abbr]
  * @param {string} [hit.alpha3]
  * @returns {number}

middleware/dedupe.js

@@ -0,0 +1,99 @@
+var logger = require('pelias-logger').get('api');
+var _ = require('lodash');
+
+function setup() {
+  return dedupeResults;
+}
+
+function dedupeResults(req, res, next) {
+  // do nothing if no result data set
+  if (_.isUndefined(req.clean) || _.isUndefined(res) || _.isUndefined(res.data)) {
+    return next();
+  }
+
+  // loop through data items and only copy unique items to uniqueResults
+  var uniqueResults = [];
+
+  _.some(res.data, function (hit) {
+    if (uniqueResults.length === 0 || _.every(uniqueResults, isDifferent.bind(null, hit)) ) {
+      uniqueResults.push(hit);
+    }
+    else {
+      logger.info('[dupe]', { query: req.clean.text, hit: hit.name.default });
+    }
+
+    // stop looping when requested size has been reached in uniqueResults
+    return req.clean.size <= uniqueResults.length;
+  });
+
+  res.data = uniqueResults;
+
+  next();
+}
+
+/**
+ * @param {object} item1
+ * @param {object} item2
+ * @returns {boolean}
+ * @throws {Error}
+ */
+function isDifferent(item1, item2) {
+  try {
+    propMatch(item1, item2, 'admin1_abbr');
+    propMatch(item1, item2, 'alpha3');
+
+    if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
+      propMatch(item1.name, item2.name, 'default');
+    }
+    else {
+      propMatch(item1, item2, 'name');
+    }
+
+    if (item1.hasOwnProperty('address') && item2.hasOwnProperty('address')) {
+      propMatch(item1.address, item2.address, 'number');
+      propMatch(item1.address, item2.address, 'street');
+      propMatch(item1.address, item2.address, 'zip');
+    }
+    else if (item1.address !== item2.address) {
+      throw new Error('different');
+    }
+  }
+  catch (err) {
+    if (err.message === 'different') {
+      return true;
+    }
+    throw err;
+  }
+
+  return false;
+}
+
+/**
+ * Throw exception if properties are different
+ *
+ * @param {object} item1
+ * @param {object} item2
+ * @param {string} prop
+ * @throws {Error}
+ */
+function propMatch(item1, item2, prop) {
+  if (normalizeString(item1[prop]) !== normalizeString(item2[prop])) {
+    throw new Error('different');
+  }
+}
+
+/**
+ * Remove punctuation and lowercase
+ *
+ * @param {string} str
+ * @returns {string}
+ */
+function normalizeString(str) {
+  if (!str) {
+    return '';
+  }
+  return _.words(str.toLowerCase()).join(' ');
+}
+
+
+module.exports = setup;

middleware/sizeCalculator.js

@@ -0,0 +1,35 @@
+var _ = require('lodash');
+
+var SIZE_PADDING = 2;
+
+/**
+ * Utility for calculating query result size
+ * incorporating padding for dedupe process
+ */
+function setup() {
+ return function setQuerySize(req, res, next) {
+   if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) {
+     return next();
+   }
+
+   req.clean.querySize = calculateSize(req.clean.size);
+   next();
+ };
+}
+
+/**
+ * Add padding or set to 1
+ *
+ * @param {number} cleanSize
+ * @returns {number}
+ */
+function calculateSize(cleanSize) {
+  switch (cleanSize || 1) {
+    case 1:
+      return 1;
+    default:
+      return cleanSize * SIZE_PADDING;
+  }
+}
+
+module.exports = setup;

package.json

@@ -33,7 +33,7 @@
     "elasticsearch": ">=1.2.1"
   },
   "dependencies": {
-    "addressit": "1.3.0",
+    "addressit": "git://github.com/dianashk/addressit.git#temp",
     "async": "^0.9.0",
     "check-types": "^3.3.1",
     "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",

query/autocomplete.js

@@ -30,9 +30,6 @@ function generateQuery( clean ){
   // input text
   vs.var( 'input:name', clean.text );
 
-  // always 10 (not user definable due to caching)
-  vs.var( 'size', 10 );
-
   // focus point
   if( check.number(clean['focus.point.lat']) &&
       check.number(clean['focus.point.lon']) ){

query/autocomplete_defaults.js

@@ -1,10 +1,10 @@
 
-var peliasQuery = require('pelias-query'),
+var peliasQuery = require('pelias-query');
-    extend = require('extend');
+var _ = require('lodash');
 
-module.exports = extend( false, peliasQuery.defaults, {
+module.exports = _.merge({}, peliasQuery.defaults, {
 
-  'size': 10,
+  'size': 20,
   'track_scores': true,
 
   'centroid:field': 'center_point',

query/reverse.js

@@ -22,15 +22,9 @@ function generateQuery( clean ){
 
   var vs = new peliasQuery.Vars( defaults );
 
-  // set defaults
-  vs.set({
-    'size': 1,
-    'boundary:circle:radius': '500km'
-  });
-
   // set size
-  if( clean.size ){
+  if( clean.querySize ){
-    vs.var( 'size', clean.size );
+    vs.var( 'size', clean.querySize);
   }
 
   // focus point to score by distance

query/reverse_defaults.js

@@ -1,10 +1,10 @@
 
-var peliasQuery = require('pelias-query'),
+var peliasQuery = require('pelias-query');
-    extend = require('extend');
+var _ = require('lodash');
 
-module.exports = extend( false, peliasQuery.defaults, {
+module.exports = _.merge({}, peliasQuery.defaults, {
 
-  'size': 10,
+  'size': 1,
   'track_scores': true,
 
   'centroid:field': 'center_point',
@@ -12,7 +12,7 @@ module.exports = extend( false, peliasQuery.defaults, {
   'sort:distance:order': 'asc',
   'sort:distance:distance_type': 'plane',
 
-  'boundary:circle:radius': '50km',
+  'boundary:circle:radius': '500km',
   'boundary:circle:distance_type': 'plane',
   'boundary:circle:optimize_bbox': 'indexed',
   'boundary:circle:_cache': true,

query/search.js

@@ -52,8 +52,8 @@ function generateQuery( clean ){
   vs.var( 'input:name', clean.text );
 
   // size
-  if( clean.size ){
+  if( clean.querySize ) {
-    vs.var( 'size', clean.size );
+    vs.var( 'size', clean.querySize );
   }
 
   // focus point

query/search_defaults.js

@@ -1,10 +1,10 @@
 
-var peliasQuery = require('pelias-query'),
+var peliasQuery = require('pelias-query');
-    extend = require('extend');
+var _ = require('lodash');
 
-module.exports = extend( false, peliasQuery.defaults, {
+module.exports = _.merge({}, peliasQuery.defaults, {
 
-  'size': 10,
+  'size': 20,
   'track_scores': true,
 
   'centroid:field': 'center_point',

routes/v1.js

@@ -12,7 +12,8 @@ var sanitisers = {
 
 /** ----------------------- middleware ------------------------ **/
 var middleware = {
-  types: require('../middleware/_types')
+  types: require('../middleware/_types'),
+  calcSize: require('../middleware/sizeCalculator')
 };
 
 /** ----------------------- controllers ----------------------- **/
@@ -30,6 +31,7 @@ var postProc = {
   distances: require('../middleware/distance'),
   confidenceScores: require('../middleware/confidenceScore'),
   confidenceScoresReverse: require('../middleware/confidenceScoreReverse'),
+  dedupe: require('../middleware/dedupe'),
   localNamingConventions: require('../middleware/localNamingConventions'),
   renamePlacenames: require('../middleware/renamePlacenames'),
   geocodeJSON: require('../middleware/geocodeJSON'),
@@ -58,9 +60,11 @@ function addRoutes(app, peliasConfig) {
     search: createRouter([
       sanitisers.search.middleware,
       middleware.types,
+      middleware.calcSize(),
       controllers.search(),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
+      postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.geocodeJSON(peliasConfig, base),
@@ -72,6 +76,7 @@ function addRoutes(app, peliasConfig) {
       controllers.search(null, require('../query/autocomplete')),
       postProc.distances('focus.point.'),
       postProc.confidenceScores(peliasConfig),
+      postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.geocodeJSON(peliasConfig, base),
@@ -80,11 +85,13 @@ function addRoutes(app, peliasConfig) {
     reverse: createRouter([
       sanitisers.reverse.middleware,
       middleware.types,
+      middleware.calcSize(),
       controllers.search(undefined, reverseQuery),
       postProc.distances('point.'),
       // reverse confidence scoring depends on distance from origin
       //  so it must be calculated first
       postProc.confidenceScoresReverse(),
+      postProc.dedupe(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.geocodeJSON(peliasConfig, base),

test/ciao/search/address_parsing.coffee

@@ -34,7 +34,7 @@ json.geocoding.query['size'].should.eql 10
 
 #? address parsing
 json.geocoding.query.parsed_text['name'].should.eql '30 w 26th st'
-json.geocoding.query.parsed_text['number'].should.eql 30
+json.geocoding.query.parsed_text['number'].should.eql '30'
 json.geocoding.query.parsed_text['street'].should.eql 'w 26th st'
 json.geocoding.query.parsed_text['state'].should.eql 'NY'
 json.geocoding.query.parsed_text['regions'].should.eql []

test/unit/fixture/autocomplete_linguistic_focus.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -115,6 +116,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': 10,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_focus_null_island.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -115,6 +116,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': 10,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/autocomplete_linguistic_only.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/autocomplete_defaults');
 
 module.exports = {
   'query': {
@@ -87,6 +88,6 @@ module.exports = {
     }
   },
   'sort': [ '_score' ],
-  'size': 10,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/dedupe_elasticsearch_results.js

@@ -0,0 +1,273 @@
+module.exports = [
+  {
+    'center_point': {
+      'lon': -76.207456,
+      'lat': 40.039265
+    },
+    'address': {},
+    'local_admin': 'East Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'East Lampeter High School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Smoketown',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Greenland',
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'osmnode',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  {
+    'center_point': {
+      'lon': -76.207456,
+      'lat': 40.039265
+    },
+    'address': {},
+    'local_admin': 'East Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'East Lampeter, High-School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Smoketown',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Greenland',
+    'category': [
+      'education'
+    ],
+    '_id': '357321757',
+    '_type': 'osmnode',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  {
+    'center_point': {
+      'lon': -76.23246,
+      'lat': 39.99288
+    },
+    'address': {},
+    'local_admin': 'West Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'Lampeter-Strasburg High School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Lampeter',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Wheatland Mills',
+    'category': [
+      'education'
+    ],
+    '_id': '4559068',
+    '_type': 'geoname',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  {
+    'center_point': {
+      'lon': -76.20746,
+      'lat': 40.03927
+    },
+    'address': {},
+    'local_admin': 'East Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'East Lampeter High School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Smoketown',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Greenland',
+    'category': [
+      'education'
+    ],
+    '_id': '5187980',
+    '_type': 'geoname',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  {
+    'center_point': {
+      'lon': -76.232457,
+      'lat': 39.992877
+    },
+    'address': {},
+    'local_admin': 'West Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'Lampeter-Strasburg High School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Lampeter',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Wheatland Mills',
+    'category': [
+      'education'
+    ],
+    '_id': '357294404',
+    '_type': 'osmnode',
+    '_score': 1.2367082,
+    'confidence': 0.879
+  },
+  {
+    'center_point': {
+      'lon': -76.207456,
+      'lat': 40.038987
+    },
+    'address': {},
+    'local_admin': 'East Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'East Lampeter School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Smoketown',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Greenland',
+    'category': [
+      'education'
+    ],
+    '_id': '357283977',
+    '_type': 'osmnode',
+    '_score': 1.1036991,
+    'confidence': 0.664
+  },
+  {
+    'center_point': {
+      'lon': -76.20746,
+      'lat': 40.03899
+    },
+    'address': {},
+    'local_admin': 'East Lampeter',
+    'admin1_abbr': 'PA',
+    'name': {
+      'default': 'East Lampeter School'
+    },
+    'admin1': 'Pennsylvania',
+    'locality': 'Smoketown',
+    'alpha3': 'USA',
+    'admin2': 'Lancaster County',
+    'admin0': 'United States',
+    'neighborhood': 'Greenland',
+    'category': [
+      'education'
+    ],
+    '_id': '5187966',
+    '_type': 'geoname',
+    '_score': 1.1036991,
+    'confidence': 0.664
+  },
+  {
+    'center_point': {
+      'lon': -94.167445,
+      'lat': 38.762788
+    },
+    'address': {},
+    'local_admin': 'Polk',
+    'admin1_abbr': 'MO',
+    'name': {
+      'default': 'Strasburg School'
+    },
+    'admin1': 'Missouri',
+    'locality': 'Strasburg',
+    'alpha3': 'USA',
+    'admin2': 'Cass County',
+    'admin0': 'United States',
+    'category': [
+      'education'
+    ],
+    '_id': '358058986',
+    '_type': 'osmnode',
+    '_score': 1.0492544,
+    'confidence': 0.658
+  },
+  {
+    'center_point': {
+      'lon': -78.36317,
+      'lat': 38.98445
+    },
+    'address': {},
+    'admin1_abbr': 'VA',
+    'name': {
+      'default': 'Strasburg High School'
+    },
+    'admin1': 'Virginia',
+    'locality': 'Strasburg',
+    'alpha3': 'USA',
+    'admin2': 'Shenandoah County',
+    'admin0': 'United States',
+    'neighborhood': 'Strasburg Junction',
+    'category': [
+      'education'
+    ],
+    '_id': '4787978',
+    '_type': 'geoname',
+    '_score': 0.9724125,
+    'confidence': 0.649
+  },
+  {
+    'center_point': {
+      'lon': -100.16516,
+      'lat': 46.13427
+    },
+    'address': {},
+    'local_admin': 'Strasburg',
+    'admin1_abbr': 'ND',
+    'name': {
+      'default': 'Strasburg High School'
+    },
+    'admin1': 'North Dakota',
+    'locality': 'Strasburg',
+    'alpha3': 'USA',
+    'admin2': 'Emmons County',
+    'admin0': 'United States',
+    'category': [
+      'education'
+    ],
+    '_id': '9683163',
+    '_type': 'geoname',
+    '_score': 0.9724125,
+    'confidence': 0.649
+  },
+  {
+    'center_point': {
+      'lon': -81.532392,
+      'lat': 40.597578
+    },
+    'address': {},
+    'local_admin': 'Franklin',
+    'admin1_abbr': 'OH',
+    'name': {
+      'default': 'Strasburg High School'
+    },
+    'admin1': 'Ohio',
+    'locality': 'Strasburg',
+    'alpha3': 'USA',
+    'admin2': 'Tuscarawas County',
+    'admin0': 'United States',
+    'category': [
+      'education'
+    ],
+    '_id': '356646971',
+    '_type': 'osmway',
+    '_score': 0.9724125,
+    'confidence': 0.649
+  }
+];

test/unit/fixture/reverse_boundary_circle.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
@@ -10,7 +11,7 @@ module.exports = {
           'must': [
             {
               'geo_distance': {
-                'distance': '500km',
+                'distance': vs.distance,
                 'distance_type': 'plane',
                 'optimize_bbox': 'indexed',
                 '_cache': true,
@@ -38,6 +39,6 @@ module.exports = {
       }
     }
   ],
-  'size': 1,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/reverse_null_island.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
@@ -40,6 +41,6 @@ module.exports = {
       }
     }
   ],
-  'size': 1,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/reverse_standard.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
@@ -40,6 +41,6 @@ module.exports = {
       }
     }
   ],
-  'size': 1,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/reverse_with_boundary_country.js

@@ -1,3 +1,4 @@
+var vs = require('../../../query/reverse_defaults');
 
 module.exports = {
   'query': {
@@ -49,6 +50,6 @@ module.exports = {
       }
     }
   ],
-  'size': 1,
+  'size': vs.size,
   'track_scores': true
 };

test/unit/fixture/search_full_address.js

@@ -1,6 +1,4 @@
-
+var vs = require('../../../query/search_defaults');
-var peliasQuery = require('pelias-query'),
-    vs = new peliasQuery.Vars( peliasQuery.defaults );
 
 module.exports = {
   'query': {
@@ -88,89 +86,89 @@ module.exports = {
           },{
             'match': {
               'address.number': {
-                'query': 123,
+                'query': '123',
-                'boost': vs.var('address:housenumber:boost').get(),
+                'boost': vs['address:housenumber:boost'],
-                'analyzer': vs.var('address:housenumber:analyzer').get()
+                'analyzer': vs['address:housenumber:analyzer']
               }
             }
           }, {
             'match': {
               'address.street': {
                 'query': 'main st',
-                'boost': vs.var('address:street:boost').get(),
+                'boost': vs['address:street:boost'],
-                'analyzer': vs.var('address:street:analyzer').get()
+                'analyzer': vs['address:street:analyzer']
               }
             }
           }, {
             'match': {
               'address.zip': {
-                'query': 10010,
+                'query': '10010',
-                'boost': vs.var('address:postcode:boost').get(),
+                'boost': vs['address:postcode:boost'],
-                'analyzer': vs.var('address:postcode:analyzer').get()
+                'analyzer': vs['address:postcode:analyzer']
               }
             }
           }, {
             'match': {
               'alpha3': {
                 'query': 'USA',
-                'boost': vs.var('admin:alpha3:boost').get(),
+                'boost': vs['admin:alpha3:boost'],
-                'analyzer': vs.var('admin:alpha3:analyzer').get()
+                'analyzer': vs['admin:alpha3:analyzer']
               }
             }
           }, {
             'match': {
               'admin0': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin0:boost').get(),
+                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs.var('admin:admin0:analyzer').get()
+                'analyzer': vs['admin:admin0:analyzer']
               }
             }
           }, {
             'match': {
               'admin1': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin1:boost').get(),
+                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs.var('admin:admin1:analyzer').get()
+                'analyzer': vs['admin:admin1:analyzer']
               }
             }
           }, {
             'match': {
               'admin1_abbr': {
                 'query': 'NY',
-                'boost': vs.var('admin:admin1_abbr:boost').get(),
+                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs.var('admin:admin1_abbr:analyzer').get()
+                'analyzer': vs['admin:admin1_abbr:analyzer']
               }
             }
           }, {
             'match': {
               'admin2': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin2:boost').get(),
+                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs.var('admin:admin2:analyzer').get()
+                'analyzer': vs['admin:admin2:analyzer']
               }
             }
           }, {
             'match': {
               'local_admin': {
                 'query': 'new york',
-                'boost': vs.var('admin:local_admin:boost').get(),
+                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs.var('admin:local_admin:analyzer').get()
+                'analyzer': vs['admin:local_admin:analyzer']
               }
             }
           }, {
             'match': {
               'locality': {
                 'query': 'new york',
-                'boost': vs.var('admin:locality:boost').get(),
+                'boost': vs['admin:locality:boost'],
-                'analyzer': vs.var('admin:locality:analyzer').get()
+                'analyzer': vs['admin:locality:analyzer']
               }
             }
           }, {
             'match': {
               'neighborhood': {
                 'query': 'new york',
-                'boost': vs.var('admin:neighborhood:boost').get(),
+                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs.var('admin:neighborhood:analyzer').get()
+                'analyzer': vs['admin:neighborhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_partial_address.js

@@ -1,6 +1,5 @@
 
-var peliasQuery = require('pelias-query'),
+var vs = require('../../../query/search_defaults');
-    vs = new peliasQuery.Vars( peliasQuery.defaults );
 
 module.exports = {
   'query': {
@@ -88,56 +87,56 @@ module.exports = {
             'match': {
               'admin0': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin0:boost').get(),
+                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs.var('admin:admin0:analyzer').get()
+                'analyzer': vs['admin:admin0:analyzer']
               }
             }
           }, {
             'match': {
               'admin1': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin1:boost').get(),
+                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs.var('admin:admin1:analyzer').get()
+                'analyzer': vs['admin:admin1:analyzer']
               }
             }
           }, {
             'match': {
               'admin1_abbr': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin1_abbr:boost').get(),
+                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs.var('admin:admin1_abbr:analyzer').get()
+                'analyzer': vs['admin:admin1_abbr:analyzer']
               }
             }
           }, {
             'match': {
               'admin2': {
                 'query': 'new york',
-                'boost': vs.var('admin:admin2:boost').get(),
+                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs.var('admin:admin2:analyzer').get()
+                'analyzer': vs['admin:admin2:analyzer']
               }
             }
           }, {
             'match': {
               'local_admin': {
                 'query': 'new york',
-                'boost': vs.var('admin:local_admin:boost').get(),
+                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs.var('admin:local_admin:analyzer').get()
+                'analyzer': vs['admin:local_admin:analyzer']
               }
             }
           }, {
             'match': {
               'locality': {
                 'query': 'new york',
-                'boost': vs.var('admin:locality:boost').get(),
+                'boost': vs['admin:locality:boost'],
-                'analyzer': vs.var('admin:locality:analyzer').get()
+                'analyzer': vs['admin:locality:analyzer']
               }
             }
           }, {
             'match': {
               'neighborhood': {
                 'query': 'new york',
-                'boost': vs.var('admin:neighborhood:boost').get(),
+                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs.var('admin:neighborhood:analyzer').get()
+                'analyzer': vs['admin:neighborhood:analyzer']
               }
             }
           }]

test/unit/fixture/search_regions_address.js

@@ -1,6 +1,5 @@
 
-var peliasQuery = require('pelias-query'),
+var vs = require('../../../query/search_defaults');
-    vs = new peliasQuery.Vars( peliasQuery.defaults );
 
 module.exports = {
   'query': {
@@ -87,73 +86,73 @@ module.exports = {
           },{
             'match': {
               'address.number': {
-                'query': 1,
+                'query': '1',
-                'boost': vs.var('address:housenumber:boost').get(),
+                'boost': vs['address:housenumber:boost'],
-                'analyzer': vs.var('address:housenumber:analyzer').get()
+                'analyzer': vs['address:housenumber:analyzer']
               }
             }
           }, {
             'match': {
               'address.street': {
                 'query': 'water st',
-                'boost': vs.var('address:street:boost').get(),
+                'boost': vs['address:street:boost'],
-                'analyzer': vs.var('address:street:analyzer').get()
+                'analyzer': vs['address:street:analyzer']
               }
             }
           }, {
             'match': {
               'admin0': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:admin0:boost').get(),
+                'boost': vs['admin:admin0:boost'],
-                'analyzer': vs.var('admin:admin0:analyzer').get()
+                'analyzer': vs['admin:admin0:analyzer']
               }
             }
           }, {
             'match': {
               'admin1': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:admin1:boost').get(),
+                'boost': vs['admin:admin1:boost'],
-                'analyzer': vs.var('admin:admin1:analyzer').get()
+                'analyzer': vs['admin:admin1:analyzer']
               }
             }
           }, {
             'match': {
               'admin1_abbr': {
                 'query': 'NY',
-                'boost': vs.var('admin:admin1_abbr:boost').get(),
+                'boost': vs['admin:admin1_abbr:boost'],
-                'analyzer': vs.var('admin:admin1_abbr:analyzer').get()
+                'analyzer': vs['admin:admin1_abbr:analyzer']
               }
             }
           }, {
             'match': {
               'admin2': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:admin2:boost').get(),
+                'boost': vs['admin:admin2:boost'],
-                'analyzer': vs.var('admin:admin2:analyzer').get()
+                'analyzer': vs['admin:admin2:analyzer']
               }
             }
           }, {
             'match': {
               'local_admin': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:local_admin:boost').get(),
+                'boost': vs['admin:local_admin:boost'],
-                'analyzer': vs.var('admin:local_admin:analyzer').get()
+                'analyzer': vs['admin:local_admin:analyzer']
               }
             }
           }, {
             'match': {
               'locality': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:locality:boost').get(),
+                'boost': vs['admin:locality:boost'],
-                'analyzer': vs.var('admin:locality:analyzer').get()
+                'analyzer': vs['admin:locality:analyzer']
               }
             }
           }, {
             'match': {
               'neighborhood': {
                 'query': 'manhattan',
-                'boost': vs.var('admin:neighborhood:boost').get(),
+                'boost': vs['admin:neighborhood:boost'],
-                'analyzer': vs.var('admin:neighborhood:analyzer').get()
+                'analyzer': vs['admin:neighborhood:analyzer']
               }
             }
           }]

test/unit/helper/sizeCalculator.js

@@ -0,0 +1,67 @@
+
+var calcSize = require('../../../middleware/sizeCalculator.js')();
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof calcSize, 'function', 'valid function');
+    t.end();
+  });
+};
+
+module.exports.tests.valid = function(test, common) {
+  var req = { clean: {} };
+  function setup(val) {
+    if (isNaN(val)) {
+      delete req.clean.size;
+    }
+    else {
+      req.clean.size = val;
+    }
+    delete req.clean.querySize;
+  }
+
+  test('size=0', function (t) {
+    setup(0);
+    calcSize(req, {}, function () {
+      t.equal(req.clean.querySize, 1);
+      t.end();
+    });
+  });
+
+  test('size=1', function (t) {
+    setup(1);
+    calcSize(req, {}, function () {
+      t.equal(req.clean.querySize, 1);
+      t.end();
+    });
+  });
+
+  test('size=10', function (t) {
+    setup(10);
+    calcSize(req, {}, function () {
+      t.equal(req.clean.querySize, 20);
+      t.end();
+    });
+  });
+
+  test('no size', function (t) {
+    setup();
+    calcSize(req, {}, function () {
+      t.equal(req.clean.hasOwnProperty('querySize'), false);
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('sizeCalculator: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/helper/text_parser.js

@@ -85,7 +85,7 @@ module.exports.tests.parse_address = function(test, common) {
     var address = parser.get_parsed_address(query_string);
 
     t.equal(typeof address, 'object', 'valid object for the address');
-    t.equal(address.number, 123, 'parsed house number');
+    t.equal(address.number, '123', 'parsed house number');
     t.equal(address.street, 'main st', 'parsed street');
     t.deepEqual(address.regions, ['new york'], 'parsed city');
     t.equal(address.state , 'NY', 'parsed state');
@@ -96,11 +96,23 @@ module.exports.tests.parse_address = function(test, common) {
     var address = parser.get_parsed_address(query_string);
 
     t.equal(typeof address, 'object', 'valid object for the address');
-    t.equal(address.number, 123, 'parsed house number');
+    t.equal(address.number, '123', 'parsed house number');
     t.equal(address.street, 'main st', 'parsed street');
     t.deepEqual(address.regions, ['new york'], 'parsed city');
     t.equal(address.state , 'NY', 'parsed state');
-    t.equal(address.postalcode, 10010, 'parsed zip');
+    t.equal(address.postalcode, '10010', 'parsed zip is a string');
+    t.end();
+  });
+  test('valid address with leading 0s in zipcode', function(t) {
+    var query_string = '339 W Main St, Cheshire, 06410';
+    var address = parser.get_parsed_address(query_string);
+
+    console.log(address);
+
+    t.equal(typeof address, 'object', 'valid object for the address');
+    t.equal(address.street, 'W Main St', 'parsed street');
+    t.deepEqual(address.regions, ['Cheshire'], 'parsed city');
+    t.equal(address.postalcode, '06410', 'parsed zip');
     t.end();
   });
 };

test/unit/middleware/dedupe.js

@@ -0,0 +1,52 @@
+var data = require('../fixture/dedupe_elasticsearch_results');
+var dedupe = require('../../../middleware/dedupe')();
+
+module.exports.tests = {};
+
+module.exports.tests.dedupe = function(test, common) {
+  test('filter out duplicates', function(t) {
+    var req = {
+      clean: {
+        text: 'lampeter strasburg high school',
+        size: 100
+      }
+    };
+    var res = {
+      data: data
+    };
+
+    var expectedCount = 7;
+    dedupe(req, res, function () {
+      t.equal(res.data.length, expectedCount, 'results have fewer items than before');
+      t.end();
+    });
+  });
+
+  test('truncate results based on specified size', function(t) {
+    var req = {
+      clean: {
+        text: 'lampeter strasburg high school',
+        size: 3
+      }
+    };
+    var res = {
+      data: data
+    };
+
+    dedupe(req, res, function () {
+      t.equal(res.data.length, req.clean.size, 'results have fewer items than before');
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[middleware] dedupe: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/reverse.js

@@ -79,14 +79,15 @@ module.exports.tests.query = function(test, common) {
 
   test('size fuzz test', function(t) {
     // test different sizes
-    var sizes = [1,2,10,undefined,null];
+    var sizes = [1,4,20,undefined,null];
-    sizes.forEach( function( size ){
+    var expected = [1,4,20,1,1];
+    sizes.forEach( function( size, index ){
       var query = generate({
-        'point.lat': 29.49136, 'point.lon': -82.50622, size: size
+        'point.lat': 29.49136, 'point.lon': -82.50622, querySize: size
       });
 
       var compiled = JSON.parse( JSON.stringify( query ) );
-      t.equal( compiled.size, size ? size : 1, 'valid reverse query for size: '+ size);
+      t.equal( compiled.size, expected[index], 'valid reverse query for size: '+ size);
     });
     t.end();
   });

test/unit/query/search.js

@@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) {
 module.exports.tests.query = function(test, common) {
   test('valid search + focus + bbox', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
       'boundary.rect.min_lat': 47.47,
       'boundary.rect.max_lon': -61.84,
@@ -31,7 +31,7 @@ module.exports.tests.query = function(test, common) {
 
   test('valid search + bbox', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'boundary.rect.min_lat': 47.47,
       'boundary.rect.max_lon': -61.84,
       'boundary.rect.max_lat': 11.51,
@@ -48,7 +48,7 @@ module.exports.tests.query = function(test, common) {
 
   test('valid lingustic-only search', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       layers: ['test']
     });
 
@@ -61,7 +61,7 @@ module.exports.tests.query = function(test, common) {
 
   test('search search + focus', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622,
       layers: ['test']
     });
@@ -75,7 +75,7 @@ module.exports.tests.query = function(test, common) {
 
   test('search search + viewport', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49136,
       'focus.viewport.max_lat': 30.49136,
       'focus.viewport.min_lon': -87.50622,
@@ -92,7 +92,7 @@ module.exports.tests.query = function(test, common) {
 
   test('search with viewport diagonal < 1km should set scale to 1km', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'focus.viewport.min_lat': 28.49135,
       'focus.viewport.max_lat': 28.49137,
       'focus.viewport.min_lon': -87.50622,
@@ -109,7 +109,7 @@ module.exports.tests.query = function(test, common) {
 
   test('search search + focus on null island', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       'focus.point.lat': 0, 'focus.point.lon': 0,
       layers: ['test']
     });
@@ -126,7 +126,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate({ text: address,
       layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
                 'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
-      size: 10,
+      querySize: 10,
       parsed_text: parser.get_parsed_address(address),
     });
 
@@ -142,7 +142,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate({ text: partial_address,
       layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
                 'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
-      size: 10,
+      querySize: 10,
       parsed_text: parser.get_parsed_address(partial_address),
     });
 
@@ -158,7 +158,7 @@ module.exports.tests.query = function(test, common) {
     var query = generate({ text: partial_address,
       layers: [ 'geoname', 'osmnode', 'osmway', 'admin0', 'admin1', 'admin2', 'neighborhood',
         'locality', 'local_admin', 'osmaddress', 'openaddresses' ],
-      size: 10,
+      querySize: 10,
       parsed_text: parser.get_parsed_address(partial_address),
     });
 
@@ -171,7 +171,7 @@ module.exports.tests.query = function(test, common) {
 
   test('valid boundary.country search', function(t) {
     var query = generate({
-      text: 'test', size: 10,
+      text: 'test', querySize: 10,
       layers: ['test'],
       'boundary.country': 'ABC'
     });

test/unit/run.js

@@ -12,10 +12,12 @@ var tests = [
   require('./helper/text_parser'),
   require('./helper/type_mapping'),
   require('./helper/types'),
+  require('./helper/sizeCalculator'),
   require('./middleware/confidenceScore'),
   require('./middleware/confidenceScoreReverse'),
   require('./middleware/distance'),
   require('./middleware/localNamingConventions'),
+  require('./middleware/dedupe'),
   require('./query/autocomplete'),
   require('./query/autocomplete_defaults'),
   require('./query/search_defaults'),