Merge pull request #769 from pelias/interpolation

interpolation: integration v1

Dockerfile

@@ -13,7 +13,7 @@ ENV HOME=/opt/pelias
 WORKDIR ${WORK}
 ADD . ${WORK}
 
-# Build and set permissions for arbitary non-root user
+# Build and set permissions for arbitrary non-root user
 RUN npm install && \
   npm test && \
   chmod -R a+rwX .

middleware/interpolate.js

@@ -0,0 +1,145 @@
+
+var async = require('async');
+var logger = require( 'pelias-logger' ).get( 'api' );
+var service = require('../service/interpolation');
+
+/**
+example response from interpolation web service:
+{
+  type: 'Feature',
+  properties: {
+    type: 'interpolated',
+    source: 'mixed',
+    number: '17',
+    lat: -41.2887032,
+    lon: 174.767089
+  },
+  geometry: {
+    type: 'Point',
+    coordinates: [ 174.767089, -41.2887032 ]
+  }
+}
+**/
+
+function setup() {
+
+  var transport = service.search();
+  var middleware = function(req, res, next) {
+
+    // no-op, user did not request an address
+    if( !isAddressQuery( req ) ){
+      return next();
+    }
+
+    // bind parsed_text variables to function call
+    var bound = interpolate.bind( transport, req.clean.parsed_text );
+
+    // perform interpolations asynchronously for all relevant hits
+    var timer = (new Date()).getTime();
+    async.map( res.data, bound, function( err, results ){
+
+      // update res.data with the mapped values
+      if( !err ){
+        res.data = results;
+      }
+
+      // log the execution time, continue
+      logger.info( '[interpolation] [took]', (new Date()).getTime() - timer, 'ms' );
+      next();
+    });
+  };
+
+  middleware.transport = transport;
+  return middleware;
+}
+
+function interpolate( parsed_text, hit, cb ){
+
+  // no-op, this hit is not from the 'street' layer
+  // note: no network request is performed.
+  if( !hit || hit.layer !== 'street' ){
+    return cb( null, hit );
+  }
+
+  // query variables
+  var coord = hit.center_point;
+  var number = parsed_text.number;
+  var street = hit.address_parts.street || parsed_text.street;
+
+  // query interpolation service
+  this.query( coord, number, street, function( err, data ){
+
+    // an error occurred
+    // note: leave this hit unmodified
+    if( err ){
+      logger.error( '[interpolation] [error]', err );
+      return cb( null, hit );
+    }
+
+    // invalid / not useful response
+    // note: leave this hit unmodified
+    if( !data || !data.hasOwnProperty('properties') ){
+      logger.info( '[interpolation] [miss]', parsed_text );
+      return cb( null, hit );
+    }
+
+    // the interpolation service returned a valid result
+    // note: we now merge thos values with the existing 'street' record.
+    logger.info( '[interpolation] [hit]', parsed_text, data );
+
+    // safety first!
+    try {
+
+      // -- metatdata --
+      hit.layer = 'address';
+      hit.match_type = 'interpolated';
+
+      // -- name --
+      hit.name.default = data.properties.number + ' ' + hit.name.default;
+
+      // -- source --
+      var source = 'mixed';
+      if( data.properties.source === 'OSM' ){ source = 'openstreetmap'; }
+      else if( data.properties.source === 'OA' ){ source = 'openaddresses'; }
+      hit.source = source;
+
+      // -- source_id --
+      // note: interpolated values have no source_id
+      delete hit.source_id; // remove original street source_id
+      if( data.properties.hasOwnProperty( 'source_id' ) ){
+        hit.source_id = data.properties.source_id;
+      }
+
+      // -- address_parts --
+      hit.address_parts.number = data.properties.number;
+
+      // -- geo --
+      hit.center_point = {
+        lat: data.properties.lat,
+        lon: data.properties.lon
+      };
+
+      // -- bbox --
+      delete hit.bounding_box;
+
+      // return the modified hit
+      return cb( null, hit );
+
+    // a syntax error occurred in the code above (this shouldn't happen!)
+    // note: the hit object may be partially modified, could possibly be invalid
+    } catch( e ){
+      logger.error( '[interpolation] [error]', e, e.stack );
+      return cb( null, hit );
+    }
+  });
+}
+
+// boolean function to check if an address was requested by the user
+function isAddressQuery( req ){
+  return req && req.hasOwnProperty('clean') &&
+         req.clean.hasOwnProperty('parsed_text') &&
+         req.clean.parsed_text.hasOwnProperty('number') &&
+         req.clean.parsed_text.hasOwnProperty('street');
+}
+
+module.exports = setup;

package.json

@@ -53,13 +53,14 @@
     "markdown": "0.5.0",
     "morgan": "1.7.0",
     "pelias-categories": "1.1.0",
-    "pelias-config": "2.4.0",
+    "pelias-config": "2.6.0",
     "pelias-labels": "1.5.1",
     "pelias-logger": "0.1.0",
     "pelias-model": "4.4.0",
     "pelias-query": "8.12.0",
     "pelias-text-analyzer": "1.7.0",
     "stats-lite": "2.0.3",
+    "superagent": "^3.2.1",
     "through2": "^2.0.3"
   },
   "devDependencies": {
@@ -70,11 +71,12 @@
     "nsp": "^2.2.0",
     "precommit-hook": "^3.0.0",
     "proxyquire": "^1.7.10",
+    "semantic-release": "^6.3.2",
     "source-map": "^0.5.6",
     "tap-dot": "1.0.5",
     "tape": "^4.5.1",
-    "uglify-js": "^2.6.2",
+    "tmp": "0.0.31",
-    "semantic-release": "^6.3.2"
+    "uglify-js": "^2.6.2"
   },
   "pre-commit": [
     "lint",

routes/v1.js

@@ -44,6 +44,7 @@ var postProc = {
   confidenceScoresReverse: require('../middleware/confidenceScoreReverse'),
   accuracy: require('../middleware/accuracy'),
   dedupe: require('../middleware/dedupe'),
+  interpolate: require('../middleware/interpolate'),
   localNamingConventions: require('../middleware/localNamingConventions'),
   renamePlacenames: require('../middleware/renamePlacenames'),
   geocodeJSON: require('../middleware/geocodeJSON'),
@@ -86,6 +87,7 @@ function addRoutes(app, peliasConfig) {
       postProc.confidenceScores(peliasConfig),
       postProc.confidenceScoresFallback(),
       postProc.dedupe(),
+      postProc.interpolate(),
       postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
@@ -104,6 +106,7 @@ function addRoutes(app, peliasConfig) {
       postProc.confidenceScores(peliasConfig),
       postProc.confidenceScoresFallback(),
       postProc.dedupe(),
+      postProc.interpolate(),
       postProc.accuracy(),
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),

service/interpolation.js

@@ -0,0 +1,118 @@
+
+var logger = require( 'pelias-logger' ).get( 'api' ),
+    request = require( 'superagent' ),
+    peliasConfig = require( 'pelias-config' );
+
+/**
+
+  street address interpolation service client
+
+  this file provides several different 'transports' which can be used to access the interpolation
+  service, either directly from disk or via a network connnection.
+
+  the exported method for this module checks pelias-config for a configuration block such as:
+
+  "interpolation": {
+    "client": {
+      "adapter": "http",
+      "host": "http://localhost:4444"
+    }
+  }
+
+  for more info on running the service see: https://github.com/pelias/interpolation
+
+**/
+
+/**
+  NullTransport
+
+  disables the service completely
+**/
+function NullTransport(){}
+NullTransport.prototype.query = function( coord, number, street, cb ){
+  cb(); // no-op
+};
+
+/**
+  RequireTransport
+
+  allows the api to be used by simply requiring the module
+**/
+function RequireTransport( addressDbPath, streetDbPath ){
+  try {
+    var lib = require('pelias-interpolation'); // lazy load dependency
+    this.query = lib.api.search( addressDbPath, streetDbPath );
+  } catch( e ){
+    logger.error( 'RequireTransport: failed to connect to interpolation service' );
+  }
+}
+RequireTransport.prototype.query = function( coord, number, street, cb ){
+  throw new Error( 'interpolation: transport not connected' );
+};
+
+/**
+  HttpTransport
+
+  allows the api to be used via a remote web service
+**/
+function HttpTransport( host, settings ){
+  this.query = function( coord, number, street, cb ){
+    request
+      .get( host + '/search/geojson' )
+      .set( 'Accept', 'application/json' )
+      .query({ lat: coord.lat, lon: coord.lon, number: number, street: street })
+      .timeout( settings && settings.timeout || 1000 )
+      .end( function( err, res ){
+        if( err || !res ){ return cb( err ); }
+        if( 200 !== res.status ){ return cb( 'non 200 status' ); }
+        return cb( null, res.body );
+      });
+  };
+}
+HttpTransport.prototype.query = function( coord, number, street, cb ){
+  throw new Error( 'interpolation: transport not connected' );
+};
+
+/**
+  Setup
+
+  allows instantiation of transport depending on configuration and preference
+**/
+module.exports.search = function setup(){
+
+  // user config
+  var config = peliasConfig.generate();
+
+  // ensure config variables set correctly
+  if( !config.hasOwnProperty('interpolation') || !config.interpolation.hasOwnProperty('client') ){
+    logger.warn( 'interpolation: configuration not found' );
+  }
+
+  // valid configuration found
+  else {
+
+    // get adapter settings from config
+    var settings = config.interpolation.client;
+
+    // http adapter
+    if( 'http' === settings.adapter && settings.hasOwnProperty('host') ){
+      logger.info( 'interpolation: using http transport:', settings.host );
+      if( settings.hasOwnProperty('timeout') ){
+        return new HttpTransport( settings.host, { timeout: parseInt( settings.timeout, 10 ) } );
+      }
+      return new HttpTransport( settings.host );
+    }
+
+    // require adapter
+    else if( 'require' === settings.adapter ){
+      if( settings.hasOwnProperty('streetdb') && settings.hasOwnProperty('addressdb') ){
+        logger.info( 'interpolation: using require transport' );
+        return new RequireTransport( settings.addressdb, settings.streetdb );
+      }
+    }
+  }
+
+  // default adapter
+  logger.info( 'interpolation: using null transport' );
+  return new NullTransport();
+};

test/unit/middleware/interpolate.js

@@ -0,0 +1,202 @@
+
+var fs = require('fs'),
+    tmp = require('tmp'),
+    setup = require('../../../middleware/interpolate');
+
+// load middleware using the default pelias config
+var load = function(){
+  // adapter is driven by config
+  var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+  fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
+  process.env.PELIAS_CONFIG = tmpfile;
+  var middleware = setup();
+  delete process.env.PELIAS_CONFIG;
+  return middleware;
+};
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    var middleware = load();
+    t.equal(typeof middleware, 'function', 'middleware is a function');
+    t.equal(middleware.length, 3, 'middleware is a function');
+    t.end();
+  });
+};
+
+module.exports.tests.isAddressQuery = function(test, common) {
+  test('invalid address query - no parsed text', function(t) {
+    var req = { clean: {} };
+
+    var middleware = load();
+    middleware(req, null, t.end);
+  });
+
+  test('invalid address query - no number', function(t) {
+    var req = { clean: {
+      parsed_text: {
+        street: 'sesame st'
+      }}
+    };
+
+    var middleware = load();
+    middleware(req, null, t.end);
+  });
+
+  test('invalid address query - no street', function(t) {
+    var req = { clean: {
+      parsed_text: {
+        number: '1',
+      }}
+    };
+
+    var middleware = load();
+    middleware(req, null, t.end);
+  });
+};
+
+// test results are correctly mapped to the transport
+module.exports.tests.map = function(test, common) {
+  test('documents mapped to transport: no hits', function(t) {
+    var req = { clean: {
+      parsed_text: {
+        number: '1',
+        street: 'sesame st'
+      }}
+    };
+    var res = { data: [] };
+
+    var middleware = load();
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [] } );
+      t.end();
+    });
+  });
+  test('documents mapped to transport: no street layer hits', function(t) {
+    var req = { clean: {
+      parsed_text: {
+        number: '1',
+        street: 'sesame st'
+      }}
+    };
+    var res = { data: [{ layer: 'foo' }] };
+
+    var middleware = load();
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [{ layer: 'foo' }] } );
+      t.end();
+    });
+  });
+};
+
+// check the service is called and response mapped correctly
+module.exports.tests.miss = function(test, common) {
+  test('miss', function(t) {
+
+    var req = { clean: {
+      parsed_text: {
+        number: '1',
+        street: 'sesame st'
+      }}
+    };
+    var res = { data: [
+      {
+        layer: 'street',
+        center_point: { lat: 1, lon: 1 },
+        address_parts: { street: 'sesame rd' },
+        name: { default: 'example' }
+      }
+    ]};
+
+    var middleware = load();
+
+    // mock out the transport
+    middleware.transport.query = function mock( coord, number, street, cb ){
+      t.deepEqual( coord, res.data[0].center_point );
+      t.deepEqual( number, req.clean.parsed_text.number );
+      t.deepEqual( street, res.data[0].address_parts.street );
+      t.equal( typeof cb, 'function' );
+      cb( 'error' );
+    };
+
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [
+        {
+          layer: 'street',
+          center_point: { lat: 1, lon: 1 },
+          address_parts: { street: 'sesame rd' },
+          name: { default: 'example' }
+        }
+      ]});
+      t.end();
+    });
+  });
+};
+
+// check the service is called and response mapped correctly
+module.exports.tests.hit = function(test, common) {
+  test('hit', function(t) {
+
+    var req = { clean: {
+      parsed_text: {
+        number: '1',
+        street: 'sesame st'
+      }}
+    };
+    var res = { data: [
+      {
+        layer: 'street',
+        center_point: { lat: 1, lon: 1 },
+        address_parts: { street: 'sesame rd' },
+        name: { default: 'street name' },
+        source_id: '123456'
+      }
+    ]};
+
+    var middleware = load();
+
+    // mock out the transport
+    middleware.transport.query = function mock( coord, number, street, cb ){
+      t.deepEqual( coord, res.data[0].center_point );
+      t.deepEqual( number, req.clean.parsed_text.number );
+      t.deepEqual( street, res.data[0].address_parts.street );
+      t.equal( typeof cb, 'function' );
+      cb( null, {
+        properties: {
+          number: '100A',
+          source: 'OSM',
+          source_id: 'way:111111',
+          lat: 22.2,
+          lon: -33.3,
+        }
+      });
+    };
+
+    middleware(req, res, function(){
+      t.deepEqual( res, { data: [
+        {
+          layer: 'address',
+          match_type: 'interpolated',
+          center_point: { lat: 22.2, lon: -33.3 },
+          address_parts: { street: 'sesame rd', number: '100A' },
+          name: { default: '100A street name' },
+          source: 'openstreetmap',
+          source_id: 'way:111111'
+        }
+      ]});
+      t.end();
+    });
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[middleware] interpolate: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/run.js

@@ -25,6 +25,7 @@ var tests = [
   require('./middleware/confidenceScoreFallback'),
   require('./middleware/confidenceScoreReverse'),
   require('./middleware/distance'),
+  require('./middleware/interpolate'),
   require('./middleware/localNamingConventions'),
   require('./middleware/dedupe'),
   require('./middleware/parseBBox'),
@@ -72,7 +73,8 @@ var tests = [
   require('./sanitizer/search_fallback'),
   require('./sanitizer/wrap'),
   require('./service/mget'),
-  require('./service/search')
+  require('./service/search'),
+  require('./service/interpolation')
 ];
 
 tests.map(function(t) {

test/unit/service/interpolation.js

@@ -0,0 +1,128 @@
+
+var fs = require('fs'),
+    tmp = require('tmp'),
+    setup = require('../../../service/interpolation').search;
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('valid interface', function(t) {
+    t.equal(typeof setup, 'function', 'setup is a function');
+    t.end();
+  });
+};
+
+// adapter factory
+module.exports.tests.factory = function(test, common) {
+
+  test('http adapter', function(t) {
+    var config = { interpolation: { client: {
+      adapter: 'http',
+      host: 'http://example.com'
+    }}};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'HttpTransport', 'HttpTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+  test('require adapter', function(t) {
+    var config = { interpolation: { client: {
+      adapter: 'require',
+      addressdb: '/tmp/address.db',
+      streetdb: '/tmp/street.db'
+    }}};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'RequireTransport', 'RequireTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+  test('null adapter', function(t) {
+    var config = { interpolation: { client: {
+      adapter: 'null'
+    }}};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+  test('default adapter', function(t) {
+    var config = {};
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, JSON.stringify( config ), { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    t.equal(adapter.constructor.name, 'NullTransport', 'NullTransport');
+    t.equal(typeof adapter, 'object', 'adapter is an object');
+    t.equal(typeof adapter.query, 'function', 'query is a function');
+    t.equal(adapter.query.length, 4, 'query function signature');
+    t.end();
+  });
+
+};
+
+// null transport
+module.exports.tests.NullTransport = function(test, common) {
+
+  test('null transport', function(t) {
+
+    // adapter is driven by config
+    var tmpfile = tmp.tmpNameSync({ postfix: '.json' });
+    fs.writeFileSync( tmpfile, '{}', { encoding: 'utf8' } );
+    process.env.PELIAS_CONFIG = tmpfile;
+    var adapter = setup();
+    delete process.env.PELIAS_CONFIG;
+
+    // test null transport performs a no-op
+    adapter.query( null, null, null, function( err, res ){
+      t.equal(err, undefined, 'no-op');
+      t.equal(res, undefined, 'no-op');
+      t.end();
+    });
+  });
+
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('SERVICE interpolation', testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};