Merge pull request #363 from pelias/master

production merge

circle.yml

@@ -0,0 +1,26 @@
+machine:
+  ruby:
+    version: 2.1.2
+  node:
+    version: 0.12.2
+
+dependencies:
+  pre:
+    - echo "{\"mapzen\":{\"api_key\":{\"search.mapzen.com\":\"$SEARCH_API_KEY\",\"pelias.mapzen.com\":\"$PELIAS_API_KEY\"}}}" >~/pelias.json
+
+deployment:
+  prod:
+    branch: production
+    commands:
+      - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install
+      - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install
+      - cd pelias-deploy && bundle exec rake deploy:api[prod]
+      - cd acceptance-tests && npm test -- -e prod
+      - cd pelias-deploy && bundle exec rake deploy:api[prod_build]
+  dev:
+    branch: master
+    commands:
+      - git clone git@github.com:pelias/acceptance-tests && cd acceptance-tests && npm install
+      - git clone git@github.com:mapzen/pelias-deploy.git && cd pelias-deploy && bundle install
+      - cd pelias-deploy && bundle exec rake deploy:api[dev]
+      - cd acceptance-tests && npm test -- -e dev

helper/adminFields.js

@@ -10,7 +10,8 @@ var ADMIN_FIELDS = [
   'admin2',
   'local_admin',
   'locality',
-  'neighborhood'
+  'neighborhood',
+  'address.zip'
 ];
 
 /**

helper/text_parser.js

@@ -8,6 +8,10 @@ var logger     = require('pelias-logger').get('api');
 
 module.exports = {};
 
+/*
+ * For performance, and to prefer POI and admin records, express a preference
+ * to only search coarse layers on very short text inputs.
+ */
 module.exports.get_layers = function get_layers(query) {
   if (query.length <= 3 ) {
     // no address parsing required

helper/types.js

@@ -2,13 +2,15 @@ var type_mapping = require( '../helper/type_mapping' );
 var _ = require('lodash');
 
 /**
- * Combine all types and determine the unique subset
+ * Different parts of the code express "preferences" for which Elasticsearch types are going to be searched
+ * This method decides how to combine all the preferences.
  *
  * @param {Array} clean_types
  * @returns {Array}
  */
 module.exports = function calculate_types(clean_types) {
-  if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_address_parser)) {
+  //Check that at least one preference of types is defined
+  if (!clean_types || !(clean_types.from_layers || clean_types.from_sources || clean_types.from_text_parser)) {
     throw new Error('clean_types should not be null or undefined');
   }
 
@@ -33,8 +35,8 @@ module.exports = function calculate_types(clean_types) {
    * Type restrictions requested by the address parser should only be used
    * if both the source and layers parameters are empty, so do this last
    */
-  if (clean_types.from_address_parser) {
-    return clean_types.from_address_parser;
+  if (clean_types.from_text_parser) {
+    return clean_types.from_text_parser;
   }
 
   throw new Error('no types specified');

package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "start": "node index.js",
     "test": "npm run unit",
-    "unit": "node test/unit/run.js | tap-spec",
+    "unit": "node test/unit/run.js | tap-dot",
     "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao",
     "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
     "audit": "npm shrinkwrap; node node_modules/nsp/bin/nspCLI.js audit-shrinkwrap; rm npm-shrinkwrap.json;",
@@ -64,7 +64,7 @@
     "nsp": "^0.3.0",
     "precommit-hook": "^1.0.7",
     "proxyquire": "^1.4.0",
-    "tap-spec": "^0.2.0",
+    "tap-dot": "^1.0.0",
     "tape": "^2.13.4"
   }
 }

query/autocomplete.js

@@ -1,6 +1,6 @@
 
 var peliasQuery = require('pelias-query'),
-    defaults = require('./defaults'),
+    defaults = require('./autocomplete_defaults'),
     check = require('check-types');
 
 //------------------------------

query/autocomplete_defaults.js

@@ -0,0 +1,95 @@
+
+var peliasQuery = require('pelias-query'),
+    extend = require('extend');
+
+module.exports = extend( false, peliasQuery.defaults, {
+
+  'size': 10,
+  'track_scores': true,
+
+  'centroid:field': 'center_point',
+
+  'sort:distance:order': 'asc',
+  'sort:distance:distance_type': 'plane',
+
+  'boundary:circle:radius': '50km',
+  'boundary:circle:distance_type': 'plane',
+  'boundary:circle:optimize_bbox': 'indexed',
+  'boundary:circle:_cache': true,
+
+  'boundary:rect:type': 'indexed',
+  'boundary:rect:_cache': true,
+
+  'ngram:analyzer': 'peliasOneEdgeGram',
+  'ngram:field': 'name.default',
+  'ngram:boost': 1,
+
+  'phrase:analyzer': 'peliasPhrase',
+  'phrase:field': 'phrase.default',
+  'phrase:boost': 1,
+  'phrase:slop': 2,
+
+  'focus:function': 'linear',
+  'focus:offset': '1km',
+  'focus:scale': '50km',
+  'focus:decay': 0.5,
+  'focus:weight': 2,
+
+  'function_score:score_mode': 'avg',
+  'function_score:boost_mode': 'replace',
+
+  'address:housenumber:analyzer': 'peliasHousenumber',
+  'address:housenumber:field': 'address.number',
+  'address:housenumber:boost': 2,
+
+  'address:street:analyzer': 'peliasStreet',
+  'address:street:field': 'address.street',
+  'address:street:boost': 5,
+
+  'address:postcode:analyzer': 'peliasZip',
+  'address:postcode:field': 'address.zip',
+  'address:postcode:boost': 20,
+
+  'admin:alpha3:analyzer': 'standard',
+  'admin:alpha3:field': 'alpha3',
+  'admin:alpha3:boost': 5,
+
+  'admin:admin0:analyzer': 'peliasAdmin',
+  'admin:admin0:field': 'admin0',
+  'admin:admin0:boost': 4,
+
+  'admin:admin1:analyzer': 'peliasAdmin',
+  'admin:admin1:field': 'admin1',
+  'admin:admin1:boost': 3,
+
+  'admin:admin1_abbr:analyzer': 'peliasAdmin',
+  'admin:admin1_abbr:field': 'admin1_abbr',
+  'admin:admin1_abbr:boost': 3,
+
+  'admin:admin2:analyzer': 'peliasAdmin',
+  'admin:admin2:field': 'admin2',
+  'admin:admin2:boost': 2,
+
+  'admin:local_admin:analyzer': 'peliasAdmin',
+  'admin:local_admin:field': 'local_admin',
+  'admin:local_admin:boost': 1,
+
+  'admin:locality:analyzer': 'peliasAdmin',
+  'admin:locality:field': 'locality',
+  'admin:locality:boost': 1,
+
+  'admin:neighborhood:analyzer': 'peliasAdmin',
+  'admin:neighborhood:field': 'neighborhood',
+  'admin:neighborhood:boost': 1,
+
+  'popularity:field': 'popularity',
+  'popularity:modifier': 'log1p',
+  'popularity:max_boost': 20,
+  'popularity:weight': 1,
+
+  'population:field': 'population',
+  'population:modifier': 'log1p',
+  'population:max_boost': 20,
+  'population:weight': 2
+
+});

query/reverse.js

@@ -1,5 +1,5 @@
 var peliasQuery = require('pelias-query'),
-    defaults = require('./defaults'),
+    defaults = require('./reverse_defaults'),
     check = require('check-types');
 
 //------------------------------

query/reverse_defaults.js

@@ -38,15 +38,15 @@ module.exports = extend( false, peliasQuery.defaults, {
   'function_score:score_mode': 'avg',
   'function_score:boost_mode': 'replace',
 
-  'address:housenumber:analyzer': 'standard',
+  'address:housenumber:analyzer': 'peliasHousenumber',
   'address:housenumber:field': 'address.number',
   'address:housenumber:boost': 2,
 
-  'address:street:analyzer': 'standard',
+  'address:street:analyzer': 'peliasStreet',
   'address:street:field': 'address.street',
   'address:street:boost': 5,
 
-  'address:postcode:analyzer': 'standard',
+  'address:postcode:analyzer': 'peliasZip',
   'address:postcode:field': 'address.zip',
   'address:postcode:boost': 3,
 

query/search.js

@@ -1,5 +1,5 @@
 var peliasQuery = require('pelias-query'),
-    defaults = require('./defaults'),
+    defaults = require('./search_defaults'),
     textParser = require('./text_parser'),
     check = require('check-types'),
     geolib = require('geolib');

query/search_defaults.js

@@ -0,0 +1,95 @@
+
+var peliasQuery = require('pelias-query'),
+    extend = require('extend');
+
+module.exports = extend( false, peliasQuery.defaults, {
+
+  'size': 10,
+  'track_scores': true,
+
+  'centroid:field': 'center_point',
+
+  'sort:distance:order': 'asc',
+  'sort:distance:distance_type': 'plane',
+
+  'boundary:circle:radius': '50km',
+  'boundary:circle:distance_type': 'plane',
+  'boundary:circle:optimize_bbox': 'indexed',
+  'boundary:circle:_cache': true,
+
+  'boundary:rect:type': 'indexed',
+  'boundary:rect:_cache': true,
+
+  'ngram:analyzer': 'peliasOneEdgeGram',
+  'ngram:field': 'name.default',
+  'ngram:boost': 1,
+
+  'phrase:analyzer': 'peliasPhrase',
+  'phrase:field': 'phrase.default',
+  'phrase:boost': 1,
+  'phrase:slop': 2,
+
+  'focus:function': 'linear',
+  'focus:offset': '1km',
+  'focus:scale': '50km',
+  'focus:decay': 0.5,
+  'focus:weight': 2,
+
+  'function_score:score_mode': 'avg',
+  'function_score:boost_mode': 'replace',
+
+  'address:housenumber:analyzer': 'peliasHousenumber',
+  'address:housenumber:field': 'address.number',
+  'address:housenumber:boost': 2,
+
+  'address:street:analyzer': 'peliasStreet',
+  'address:street:field': 'address.street',
+  'address:street:boost': 5,
+
+  'address:postcode:analyzer': 'peliasZip',
+  'address:postcode:field': 'address.zip',
+  'address:postcode:boost': 20,
+
+  'admin:alpha3:analyzer': 'standard',
+  'admin:alpha3:field': 'alpha3',
+  'admin:alpha3:boost': 5,
+
+  'admin:admin0:analyzer': 'peliasAdmin',
+  'admin:admin0:field': 'admin0',
+  'admin:admin0:boost': 4,
+
+  'admin:admin1:analyzer': 'peliasAdmin',
+  'admin:admin1:field': 'admin1',
+  'admin:admin1:boost': 3,
+
+  'admin:admin1_abbr:analyzer': 'peliasAdmin',
+  'admin:admin1_abbr:field': 'admin1_abbr',
+  'admin:admin1_abbr:boost': 3,
+
+  'admin:admin2:analyzer': 'peliasAdmin',
+  'admin:admin2:field': 'admin2',
+  'admin:admin2:boost': 2,
+
+  'admin:local_admin:analyzer': 'peliasAdmin',
+  'admin:local_admin:field': 'local_admin',
+  'admin:local_admin:boost': 1,
+
+  'admin:locality:analyzer': 'peliasAdmin',
+  'admin:locality:field': 'locality',
+  'admin:locality:boost': 1,
+
+  'admin:neighborhood:analyzer': 'peliasAdmin',
+  'admin:neighborhood:field': 'neighborhood',
+  'admin:neighborhood:boost': 1,
+
+  'popularity:field': 'popularity',
+  'popularity:modifier': 'log1p',
+  'popularity:max_boost': 20,
+  'popularity:weight': 1,
+
+  'population:field': 'population',
+  'population:modifier': 'log1p',
+  'population:max_boost': 20,
+  'population:weight': 2
+
+});

sanitiser/_geo_reverse.js

@@ -1,7 +1,7 @@
 
 var geo_common = require ('./_geo_common');
 var _ = require('lodash');
-var defaults = require('../query/defaults');
+var defaults = require('../query/reverse_defaults');
 var LAT_LON_IS_REQUIRED = true,
     CIRCLE_IS_REQUIRED = false;
 

sanitiser/_text.js

@@ -1,5 +1,5 @@
 var check = require('check-types'),
-    query_parser = require('../helper/query_parser');
+    text_parser = require('../helper/text_parser');
 
 // validate texts, convert types and apply defaults
 function sanitize( raw, clean ){
@@ -19,14 +19,14 @@ function sanitize( raw, clean ){
     clean.text = raw.text;
 
     // parse text with query parser
-    var parsed_text = query_parser.get_parsed_address(clean.text);
+    var parsed_text = text_parser.get_parsed_address(clean.text);
     if (check.assigned(parsed_text)) {
       clean.parsed_text = parsed_text;
     }
 
     // try to set layers from query parser results
     clean.types = clean.layers || {};
-    clean.types.from_address_parsing = query_parser.get_layers(clean.text);
+    clean.types.from_text_parser = text_parser.get_layers(clean.text);
   }
 
   return messages;

test/unit/helper/text_parser.js

@@ -1,4 +1,4 @@
-var parser = require('../../../helper/query_parser');
+var parser = require('../../../helper/text_parser');
 
 var type_mapping = require('../../../helper/type_mapping');
 var layers_map = type_mapping.layer_with_aliases_to_type;

test/unit/helper/types.js

@@ -33,7 +33,7 @@ module.exports.tests.no_cleaned_types = function(test, common) {
 module.exports.tests.address_parser = function(test, common) {
   test('address parser specifies only admin layers', function(t) {
     var cleaned_types = {
-      from_address_parser: ['admin0'] // simplified return value from address parser
+      from_text_parser: ['admin0'] // simplified return value from address parser
     };
     var actual = types(cleaned_types);
     var expected = ['admin0']; // simplified expected value for all admin layers
@@ -58,7 +58,7 @@ module.exports.tests.layers_parameter_and_address_parser = function(test, common
   test('layers parameter and address parser present', function(t) {
     var cleaned_types = {
       from_layers: ['geoname'],
-      from_address_parser: ['admin0'] // simplified return value from address parse
+      from_text_parser: ['admin0'] // simplified return value from address parse
     };
     var actual = types(cleaned_types);
     var expected = ['geoname'];

test/unit/query/autocomplete.js

@@ -1,6 +1,6 @@
 
 var generate = require('../../../query/autocomplete');
-var parser = require('../../../helper/query_parser');
+var parser = require('../../../helper/text_parser');
 
 module.exports.tests = {};
 

test/unit/query/autocomplete_defaults.js

@@ -0,0 +1,22 @@
+
+var defaults = require('../../../query/autocomplete_defaults');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof defaults, 'object', 'defaults defined');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('autocomplete defaults ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/reverse_defaults.js

@@ -0,0 +1,22 @@
+
+var defaults = require('../../../query/reverse_defaults');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof defaults, 'object', 'defaults defined');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('reverse defaults ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/query/search.js

@@ -1,5 +1,5 @@
 var generate = require('../../../query/search');
-var parser = require('../../../helper/query_parser');
+var parser = require('../../../helper/text_parser');
 
 module.exports.tests = {};
 

test/unit/query/search_defaults.js

@@ -1,5 +1,5 @@
 
-var defaults = require('../../../query/defaults');
+var defaults = require('../../../query/search_defaults');
 
 module.exports.tests = {};
 
@@ -13,7 +13,7 @@ module.exports.tests.interface = function(test, common) {
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
-    return tape('query defaults ' + name, testFunction);
+    return tape('search defaults ' + name, testFunction);
   }
 
   for( var testCase in module.exports.tests ){

test/unit/run.js

@@ -9,14 +9,16 @@ var tests = [
   require('./helper/geojsonify'),
   require('./helper/labelGenerator'),
   require('./helper/labelSchema'),
-  require('./helper/query_parser'),
+  require('./helper/text_parser'),
   require('./helper/type_mapping'),
   require('./helper/types'),
   require('./middleware/confidenceScore'),
   require('./middleware/confidenceScoreReverse'),
   require('./middleware/distance'),
   require('./query/autocomplete'),
-  require('./query/defaults'),
+  require('./query/autocomplete_defaults'),
+  require('./query/search_defaults'),
+  require('./query/reverse_defaults'),
   require('./query/reverse'),
   require('./query/search'),
   require('./sanitiser/_boundary_country'),
@@ -29,6 +31,7 @@ var tests = [
   require('./sanitiser/_single_scalar_parameters'),
   require('./sanitiser/_size'),
   require('./sanitiser/_sources'),
+  require('./sanitiser/_text'),
   require('./sanitiser/autocomplete'),
   require('./sanitiser/place'),
   require('./sanitiser/reverse'),

test/unit/sanitiser/_geo_reverse.js

@@ -1,5 +1,5 @@
 var sanitize = require('../../../sanitiser/_geo_reverse');
-var defaults = require('../../../query/defaults');
+var defaults = require('../../../query/reverse_defaults');
 
 module.exports.tests = {};
 

test/unit/sanitiser/_text.js

@@ -0,0 +1,32 @@
+var sanitiser = require('../../../sanitiser/_text');
+var type_mapping = require('../../../helper/type_mapping');
+
+module.exports.tests = {};
+
+module.exports.tests.text_parser = function(test, common) {
+  test('short input text has admin layers set ', function(t) {
+    var raw = {
+      text: 'emp'  //start of empire state building
+    };
+    var clean = {
+    };
+
+    var messages = sanitiser(raw, clean);
+
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+    t.equal(clean.types.from_text_parser, type_mapping.layer_with_aliases_to_type.coarse, 'coarse layers preferred');
+
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+  function test(name, testFunction) {
+    return tape('SANITISER _text: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/sanitiser/reverse.js

@@ -4,7 +4,7 @@
 var reverse  = require('../../../sanitiser/reverse'),
     sanitize = reverse.sanitize,
     middleware = reverse.middleware,
-    defaults = require('../../../query/defaults'),
+    defaults = require('../../../query/reverse_defaults'),
     defaultError = 'missing param \'lat\'',
     defaultClean =  { 'point.lat': 0,
                       'point.lon': 0,

test/unit/sanitiser/search.js

@@ -1,6 +1,6 @@
 var extend = require('extend'),
     search  = require('../../../sanitiser/search'),
-    parser = require('../../../helper/query_parser'),
+    parser = require('../../../helper/text_parser'),
     sanitize = search.sanitize,
     middleware = search.middleware,
     defaultError = 'invalid param \'text\': text length, must be >0';