Merge pull request #38 from pelias/suggest_plus_mget

perform mget during suggest. requires suggester output to be a gid
10 years ago · 3f7ea32ce6
19 changed files with 649 additions and 202 deletions
--- a/controller/suggest.js
+++ b/controller/suggest.js
@ -1,6 +1,9 @@

-var service = { suggest: require('../service/suggest') };
-var geojsonify = require('../helper/geojsonify').suggest;
+var service = {
+  suggest: require('../service/suggest'),
+  mget: require('../service/mget')
+};
+var geojsonify = require('../helper/geojsonify').search;
 var async = require('async');

 function setup( backend, query ){
@ -21,7 +24,7 @@ function setup( backend, query ){
    var query_backend = function(cmd, callback) {
      // query backend
      service.suggest( backend, cmd, function( err, docs ){
-        
+
        // error handler
        if( err ){ return next( err ); }

@ -32,18 +35,18 @@ function setup( backend, query ){
    var dedup = function(combined) {
      var unique_ids = [];
      return combined.filter(function(item, pos) {
-        if (unique_ids.indexOf(item.payload.id) == -1) {
-          unique_ids.push(item.payload.id);
-          return true;  
+        if (unique_ids.indexOf(item.text) == -1) {
+          unique_ids.push(item.text);
+          return true;
        }
        return false;
      });
    };

-    var respond = function(data) {
-
+    var reply = function(docs) {
+      
      // convert docs to geojson
-      var geojson = geojsonify( data );
+      var geojson = geojsonify( docs );

      // response envelope
      geojson.date = new Date().getTime();
@ -52,12 +55,41 @@ function setup( backend, query ){
      return res.status(200).json( geojson );
    };

+    var respond = function(data) {
+
+      // no documents suggested, return empty array to avoid ActionRequestValidationException
+      if( !Array.isArray( data ) || !data.length ){
+        return reply([]);
+      }
+
+      // map suggester output to mget query
+      var query = data.map( function( doc ) {
+        var idParts = doc.text.split(':');
+        return {
+          _index: 'pelias',
+          _type: idParts[0],
+          _id: idParts.slice(1).join(':')
+        };
+      });
+
+      service.mget( backend, query, function( err, docs ){
+
+        // error handler
+        if( err ){ return next( err ); }
+
+        // reply
+        return reply( docs );
+
+      });
+
+    };
+
    if (req.clean.input) {
      var async_query;

      // admin only
      req.admin = {};
-      for (k in req.clean) { req.admin[k] = req.clean[k] }
+      for (var k in req.clean) { req.admin[k] = req.clean[k]; }
      req.admin.layers = ['admin0','admin1','admin2'];

      if (req.clean.input.length < 4 && isNaN(parseInt(req.clean.input, 10))) {
@ -74,7 +106,7 @@ function setup( backend, query ){
            cmd.body = query( req.clean, 3 );
            query_backend(cmd, callback);
          }
-        }
+        };
      } else {
        async_query = {
          all_5p: function(callback){
@ -93,7 +125,7 @@ function setup( backend, query ){
            cmd.body = query( req.admin );
            query_backend(cmd, callback);
          }
-        }
+        };
      }
      
      async.parallel(async_query, function(err, results) {
@ -101,7 +133,7 @@ function setup( backend, query ){
        var splice_length = parseInt((SIZE / Object.keys(results).length), 10);
        var results_keys = Object.keys(async_query);
        
-        var combined = []; 
+        var combined = [];
        results_keys.forEach(function(key){
          combined = combined.concat(results[key].splice(0,splice_length));
        });
--- a/controller/suggest_nearby.js
+++ b/controller/suggest_nearby.js
@ -1,6 +1,9 @@

-var service = { suggest: require('../service/suggest') };
-var geojsonify = require('../helper/geojsonify').suggest;
+var service = {
+  suggest: require('../service/suggest'),
+  mget: require('../service/mget')
+};
+var geojsonify = require('../helper/geojsonify').search;

 function setup( backend, query ){

@ -16,12 +19,9 @@ function setup( backend, query ){
      body: query( req.clean )
    };

-    // query backend
-    service.suggest( backend, cmd, function( err, docs ){
-
-      // error handler
-      if( err ){ return next( err ); }
-
+    // responder
+    function reply( docs ){
+      
      // convert docs to geojson
      var geojson = geojsonify( docs );

@ -30,6 +30,38 @@ function setup( backend, query ){

      // respond
      return res.status(200).json( geojson );
+    }
+
+    // query backend
+    service.suggest( backend, cmd, function( err, suggested ){
+
+      // error handler
+      if( err ){ return next( err ); }
+
+      // no documents suggested, return empty array to avoid ActionRequestValidationException
+      if( !Array.isArray( suggested ) || !suggested.length ){
+        return reply([]);
+      }
+
+      // map suggester output to mget query
+      var query = suggested.map( function( doc ) {
+        var idParts = doc.text.split(':');
+        return {
+          _index: 'pelias',
+          _type: idParts[0],
+          _id: idParts.slice(1).join(':')
+        };
+      });
+
+      service.mget( backend, query, function( err, docs ){
+
+        // error handler
+        if( err ){ return next( err ); }
+
+        // reply
+        return reply( docs );
+
+      });
    });

  }
--- a/helper/geojsonify.js
+++ b/helper/geojsonify.js
@ -1,7 +1,8 @@

-var GeoJSON = require('geojson');
+var GeoJSON = require('geojson'),
+    outputGenerator = require('./outputGenerator');

-function suggest( docs ){
+function search( docs ){

  // emit a warning if the doc format is invalid
  // @note: if you see this error, fix it ASAP!
@ -14,50 +15,14 @@ function suggest( docs ){
  var geodata = docs.map( function( doc ){

    // something went very wrong
-    if( !doc || !doc.payload ) return warning();
-
-    // split payload id string in to geojson properties
-    if( 'string' !== typeof doc.payload.id ) return warning();
-    var idParts = doc.payload.id.split('/');
-    doc.type = idParts[0];
-    doc.id = idParts[1];
-
-    // split payload geo string in to geojson properties
-    if( 'string' !== typeof doc.payload.geo ) return warning();
-    var geoParts = doc.payload.geo.split(',');
-    doc.lat = parseFloat( geoParts[1] );
-    doc.lng = parseFloat( geoParts[0] );
-
-    // remove payload from doc
-    delete doc.payload;
-    return doc;
-
-  // filter-out invalid entries
-  }).filter( function( doc ){
-    return doc;
-  });
-
-  // convert to geojson
-  return GeoJSON.parse( geodata, { Point: ['lat', 'lng'] } );
-
-}
-
-function search( docs ){
-
-  // emit a warning if the doc format is invalid
-  // @note: if you see this error, fix it ASAP!
-  function warning(){
-    console.error( 'error: invalid doc', __filename );
-    return false; // remove offending doc from results
-  }
-
-  // flatten & expand data for geojson conversion
-  var geodata = docs.map( function( doc ){
+    if( !doc ) return warning();

    var output = {};

-    // something went very wrong
-    if( !doc ) return warning();
+    // provide metadata to consumer
+    output.id = doc._id;
+    output.type = doc._type;
+    output.layer = doc._type;

    // map center_point
    if( !doc.center_point ) return warning();
@ -78,10 +43,8 @@ function search( docs ){
    if( doc.locality ){ output.locality = doc.locality; }
    if( doc.neighborhood ){ output.neighborhood = doc.neighborhood; }

-    // map suggest output
-    if( doc.suggest && doc.suggest.output ){
-      output.text = doc.suggest.output;
-    }
+    // generate region-specific text string
+    output.text = outputGenerator( doc );

    return output;

@ -95,5 +58,4 @@ function search( docs ){

 }

-module.exports.suggest = suggest;
 module.exports.search = search;
--- a/helper/outputGenerator.js
+++ b/helper/outputGenerator.js
@ -0,0 +1,37 @@
+
+var schemas = require('./outputSchema.json');
+
+module.exports = function( record ){
+
+  var adminParts = [];
+
+  var schema = schemas.default;
+  
+  if (record.alpha3 && record.alpha3.length && schemas[record.alpha3]) {
+    schema = schemas[record.alpha3];
+  }
+  
+  var buildOutput = function(parts, schemaArr, record) {
+    for (var i=0; i<schemaArr.length; i++) {
+      var rec = record[schemaArr[i]];
+      if (rec && rec.length) {
+        parts.push( rec );
+        return parts;
+      }
+    }
+    return parts;
+  };
+
+  for (var key in schema) {
+    adminParts = buildOutput(adminParts, schema[key], record);  
+  }
+
+  var outputs = [ record.name.default ].concat( adminParts );
+
+  // de-dupe outputs
+  outputs = outputs.filter( function( i, pos ) {
+    return outputs.indexOf( i ) == pos;
+  });
+
+  return outputs.join(', ').trim();
+};
--- a/helper/outputSchema.json
+++ b/helper/outputSchema.json
@ -0,0 +1,14 @@
+{
+  "USA": {
+    "local": ["local_admin", "locality", "neighborhood", "admin2"],
+    "regional": ["admin1_abbr", "admin1", "admin0"]
+  },
+  "GBR": {
+    "local": ["neighborhood", "admin2", "local_admin", "locality"],
+    "regional": ["admin2","admin0","admin1"]
+  },
+  "default": {
+    "local": ["local_admin", "locality", "neighborhood", "admin2"],
+    "regional": ["admin1_abbr", "admin1", "admin0"]
+  }
+}
--- a/service/mget.js
+++ b/service/mget.js
@ -29,7 +29,19 @@ function service( backend, query, cb ){
    // map returned documents
    var docs = [];
    if( data && Array.isArray(data.docs) ){
-      docs = data.docs.map( function( doc ){
+
+      docs = data.docs.filter( function( doc ){
+
+        // remove docs not actually found
+        return doc.found;
+      
+      }).map( function( doc ){
+
+        // map metadata in to _source so we
+        // can serve it up to the consumer
+        doc._source._id = doc._id;
+        doc._source._type = doc._type;
+
        return doc._source;
      });
    }
--- a/service/search.js
+++ b/service/search.js
@ -17,6 +17,12 @@ function service( backend, cmd, cb ){
    var docs = [];
    if( data && data.hits && data.hits.total && Array.isArray(data.hits.hits)){
      docs = data.hits.hits.map( function( hit ){
+
+        // map metadata in to _source so we
+        // can serve it up to the consumer
+        hit._source._id = hit._id;
+        hit._source._type = hit._type;
+
        return hit._source;
      });
    }
--- a/test/unit/controller/doc.js
+++ b/test/unit/controller/doc.js
@ -23,10 +23,14 @@ module.exports.tests.functional_success = function(test, common) {
      coordinates: [ -50.5, 100.1 ]
    },
    properties: {
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
      name: 'test name1',
      admin0: 'country1',
      admin1: 'state1',
-      admin2: 'city1'
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
    }
  }, {
    type: 'Feature',
@ -35,15 +39,19 @@ module.exports.tests.functional_success = function(test, common) {
      coordinates: [ -51.5, 100.2 ]
    },
    properties: {
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
      name: 'test name2',
      admin0: 'country2',
      admin1: 'state2',
-      admin2: 'city2'
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
    }
  }];

  test('functional success', function(t) {
-    var backend = mockBackend( 'client/doc/ok/1', function( cmd ){
+    var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
    });
    var controller = setup( backend );
@ -68,7 +76,7 @@ module.exports.tests.functional_success = function(test, common) {
 // functionally test controller (backend failure)
 module.exports.tests.functional_failure = function(test, common) {
  test('functional failure', function(t) {
-    var backend = mockBackend( 'client/doc/fail/1', function( cmd ){
+    var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'b' } ] } }, 'correct backend command');
    });
    var controller = setup( backend );
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@ -24,10 +24,14 @@ module.exports.tests.functional_success = function(test, common) {
      coordinates: [ -50.5, 100.1 ]
    },
    properties: {
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
      name: 'test name1',
      admin0: 'country1',
      admin1: 'state1',
-      admin2: 'city1'
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
    }
  }, {
    type: 'Feature',
@ -36,10 +40,14 @@ module.exports.tests.functional_success = function(test, common) {
      coordinates: [ -51.5, 100.2 ]
    },
    properties: {
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
      name: 'test name2',
      admin0: 'country2',
      admin1: 'state2',
-      admin2: 'city2'
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
    }
  }];

--- a/test/unit/controller/suggest.js
+++ b/test/unit/controller/suggest.js
@ -16,38 +16,51 @@ module.exports.tests.interface = function(test, common) {
 // functionally test controller (backend success)
 module.exports.tests.functional_success = function(test, common) {

-  // expected geojson features for 'client/suggest/ok/1' fixture
+  // expected geojson features for 'client/mget/ok/1' fixture
  var expected = [{
    type: 'Feature',
    geometry: {
      type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -50.5, 100.1 ]
    },
    properties: {
-      id: 'mockid1',
-      type: 'mocktype',
-      value: 1
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
+      name: 'test name1',
+      admin0: 'country1',
+      admin1: 'state1',
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
    }
  }, {
    type: 'Feature',
    geometry: {
      type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -51.5, 100.2 ]
    },
    properties: {
-      id: 'mockid2',
-      type: 'mocktype',
-      value: 2
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
+      name: 'test name2',
+      admin0: 'country2',
+      admin1: 'state2',
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
    }
  }];

  test('functional success', function(t) {
    var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
-      if (cmd.body.layers) {
+      // the backend executes suggest (vanilla and admin-only) and mget, so we check them all based on cmd
+      if( cmd.body.docs ){
+        t.deepEqual(cmd, { body: { docs: [ { _id: 'mockid1', _index: 'pelias', _type: 'mocktype' } , { _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }] } }, 'correct mget command');
+      } else if (cmd.body.layers) {
        // layers are set exclusively for admin: test for admin-only layers
-        t.deepEqual(cmd, { body: { input: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' }, 'correct backend command');  
+        t.deepEqual(cmd, { body: { input: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' }, 'correct suggest/admin command');
      } else {
-        t.deepEqual(cmd, { body: { input: 'b' }, index: 'pelias' }, 'correct backend command');
+        t.deepEqual(cmd, { body: { input: 'b' }, index: 'pelias' }, 'correct suggest command');
      }
    });
    var controller = setup( backend, mockQuery() );
@ -73,7 +86,14 @@ module.exports.tests.functional_success = function(test, common) {
 module.exports.tests.functional_failure = function(test, common) {
  test('functional failure', function(t) {
    var backend = mockBackend( 'client/suggest/fail/1', function( cmd ){
-      t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command');
+      if( cmd.body.docs ){
+        t.deepEqual(cmd, { body: { docs: [ { _id: 'mockid1', _index: 'pelias', _type: 'mocktype' } , { _id: 'mockid2', _index: 'pelias', _type: 'mocktype' }] } }, 'correct mget command');
+      } else if (cmd.body.layers) {
+        // layers are set exclusively for admin: test for admin-only layers
+        t.deepEqual(cmd, { body: { a: 'b', layers: [ 'admin0', 'admin1', 'admin2' ] }, index: 'pelias' }, 'correct suggest/admin command');
+      } else {
+        t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
+      }
    });
    var controller = setup( backend, mockQuery() );
    var next = function( message ){
--- a/test/unit/controller/suggest_nearby.js
+++ b/test/unit/controller/suggest_nearby.js
@ -16,34 +16,50 @@ module.exports.tests.interface = function(test, common) {
 // functionally test controller (backend success)
 module.exports.tests.functional_success = function(test, common) {

-  // expected geojson features for 'client/suggest/ok/1' fixture
+  // expected geojson features for 'client/mget/ok/1' fixture
  var expected = [{
    type: 'Feature',
    geometry: {
      type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -50.5, 100.1 ]
    },
    properties: {
-      id: 'mockid1',
-      type: 'mocktype',
-      value: 1
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
+      name: 'test name1',
+      admin0: 'country1',
+      admin1: 'state1',
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
    }
  }, {
    type: 'Feature',
    geometry: {
      type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -51.5, 100.2 ]
    },
    properties: {
-      id: 'mockid2',
-      type: 'mocktype',
-      value: 2
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
+      name: 'test name2',
+      admin0: 'country2',
+      admin1: 'state2',
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
    }
  }];

  test('functional success', function(t) {
+    var i = 0;
    var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
-      t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command');
+      // the backend executes 2 commands, so we check them both
+      if( ++i === 1 ){
+        t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
+      } else {
+        t.deepEqual(cmd, { body: { docs: [ { _id: 'mockid1', _index: 'pelias', _type: 'mocktype' }, { _id: 'mockid2', _index: 'pelias', _type: 'mocktype' } ] } }, 'correct mget command');
+      }
    });
    var controller = setup( backend, mockQuery() );
    var res = {
@ -82,7 +98,7 @@ module.exports.tests.functional_failure = function(test, common) {
 module.exports.all = function (tape, common) {

  function test(name, testFunction) {
-    return tape('GET /suggest ' + name, testFunction);
+    return tape('GET /suggest/nearby ' + name, testFunction);
  }

  for( var testCase in module.exports.tests ){
--- a/test/unit/helper/geojsonify.js
+++ b/test/unit/helper/geojsonify.js
@ -4,77 +4,19 @@ var geojsonify = require('../../../helper/geojsonify');
 module.exports.tests = {};

 module.exports.tests.interface = function(test, common) {
-  test('valid interface .suggest()', function(t) {
-    t.equal(typeof geojsonify.suggest, 'function', 'suggest is a function');
-    t.equal(geojsonify.suggest.length, 1, 'accepts x arguments');
+  test('valid interface .search()', function(t) {
+    t.equal(typeof geojsonify.search, 'function', 'search is a function');
+    t.equal(geojsonify.search.length, 1, 'accepts x arguments');
    t.end();
  });
 };

-module.exports.tests.suggest = function(test, common) {
-
-  var input = [{
-    bingo1: 'bango1',
-    payload: {
-      id: 'foo1/bar1',
-      geo: '100,-10.5'
-    }
-  },{
-    bingo2: 'bango2',
-    payload: {
-      id: 'foo2/bar2',
-      geo: '10,-1.5'
-    }
-  }];
-
-  var expected = {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            100,
-            -10.5
-          ]
-        },
-        "properties": {
-          "bingo1": "bango1",
-          "type": "foo1",
-          "id": "bar1"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            10,
-            -1.5
-          ]
-        },
-        "properties": {
-          "bingo2": "bango2",
-          "type": "foo2",
-          "id": "bar2"
-        }
-      }
-    ]
-  };
-
-  test('geojsonify.suggest()', function(t) {
-    var json = geojsonify.suggest( input );
-    t.deepEqual(json, expected, 'all docs mapped');
-    t.end();
-  });
-};
-
-
 module.exports.tests.search = function(test, common) {

  var input = [
    {
+      "_id": "id1",
+      "_type": "type1",
      "center_point": {
        "lat": 51.5337144,
        "lon": -0.1069716
@ -100,14 +42,12 @@ module.exports.tests.search = function(test, common) {
        "input": [
          "'round midnight jazz and blues bar"
        ],
-        "payload": {
-          "id": "osmnode/2208150035",
-          "geo": "-0.10697160000000001,51.53371440000001"
-        },
-        "output": "'Round Midnight Jazz and Blues Bar, Angel, United Kingdom"
+        "output": "osmnode:2208150035"
      }
    },
    {
+      "_id": "id2",
+      "_type": "type2",
      "type": "way",
      "name": {
        "default": "Blues Cafe"
@ -128,11 +68,33 @@ module.exports.tests.search = function(test, common) {
        "input": [
          "blues cafe"
        ],
-        "payload": {
-          "id": "osmway/147495160",
-          "geo": "-0.101795,51.517806"
-        },
-        "output": "Blues Cafe, Smithfield, United Kingdom"
+        "output": "osmway:147495160"
+      }
+    },
+    {
+      "_id": "34633854",
+      "_type": "osmway",
+      "type": "osmway",
+      "name": {
+        "default": "Empire State Building"
+      },
+      "center_point": {
+        "lat": "40.748432",
+        "lon": "-73.985656"
+      },
+      "alpha3": "USA",
+      "admin0": "United States",
+      "admin1": "New York",
+      "admin1_abbr": "NY",
+      "admin2": "New York",
+      "local_admin": "Manhattan",
+      "locality": "New York",
+      "neighborhood": "Koreatown",
+      "suggest": {
+        "input": [
+          "empire state building"
+        ],
+        "output": "osmway:34633854"
      }
    }
  ];
@ -150,7 +112,10 @@ module.exports.tests.search = function(test, common) {
          ]
        },
        "properties": {
-          "text": "'Round Midnight Jazz and Blues Bar, Angel, United Kingdom",
+          "id": "id1",
+          "type": "type1",
+          "layer": "type1",
+          "text": "'Round Midnight Jazz and Blues Bar, test3, Angel",
          "name": "'Round Midnight Jazz and Blues Bar",
          "alpha3": "GBR",
          "admin0": "United Kingdom",
@ -159,7 +124,7 @@ module.exports.tests.search = function(test, common) {
          "admin2": "Angel",
          "local_admin": "test1",
          "locality": "test2",
-          "neighborhood": "test3",
+          "neighborhood": "test3"
        }
      },
      {
@ -172,7 +137,10 @@ module.exports.tests.search = function(test, common) {
          ]
        },
        "properties": {
-          "text": "Blues Cafe, Smithfield, United Kingdom",
+          "id": "id2",
+          "type": "type2",
+          "layer": "type2",
+          "text": "Blues Cafe, test3, Smithfield",
          "name": "Blues Cafe",
          "alpha3": "GBR",
          "admin0": "United Kingdom",
@ -181,7 +149,32 @@ module.exports.tests.search = function(test, common) {
          "admin2": "Smithfield",
          "local_admin": "test1",
          "locality": "test2",
-          "neighborhood": "test3",
+          "neighborhood": "test3"
+        }
+      },
+      {
+        "type": "Feature",
+        "geometry": {
+          "type": "Point",
+          "coordinates": [
+            -73.985656,
+            40.748432
+          ]
+        },
+        "properties": {
+          "id": "34633854",
+          "type": "osmway",
+          "layer": "osmway",
+          "text": "Empire State Building, Manhattan, NY",
+          "name": "Empire State Building",
+          "alpha3": "USA",
+          "admin0": "United States",
+          "admin1": "New York",
+          "admin1_abbr": "NY",
+          "admin2": "New York",
+          "local_admin": "Manhattan",
+          "locality": "New York",
+          "neighborhood": "Koreatown"
        }
      }
    ]
--- a/test/unit/helper/outputSchema.js
+++ b/test/unit/helper/outputSchema.js
@ -0,0 +1,55 @@
+
+var schemas = require('../../../helper/outputSchema.json');
+var alpha3  = require('../mock/alpha3.json');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof schemas, 'object', 'valid object');
+    t.equal(schemas.hasOwnProperty('default'), true, 'has default defined');
+    t.end();
+  });
+};
+
+module.exports.tests.valid = function(test, common) {
+  var valid_keys = ['local_admin', 'locality', 'neighborhood', 'admin2', 'admin1_abbr', 'admin1', 'admin0'];
+  var default_schema = {
+    local: ['local_admin', 'locality', 'neighborhood', 'admin2'],
+    regional: ['admin1_abbr', 'admin1', 'admin0']
+  };
+
+  var isValid = function(keys, schema) {
+    test('valid key/object (' + keys + ')' , function(t) {
+      if (keys=="default") { 
+        t.deepEqual(schema, default_schema, 'valid default schema');
+      } else {
+        t.equal(alpha3.hasOwnProperty(keys), true, 'valid key');  
+      }
+      t.equal(typeof schema, 'object', 'valid object');
+      t.notEqual(Object.getOwnPropertyNames(schema).length, 0,  'object not empty');
+      for (levels in schema) {
+        t.equal(Object.prototype.toString.call(schema[levels]), '[object Array]', levels+' is an array');
+        for (var i=0;i<schema[levels].length;i++) {
+          var key = schema[levels][i];
+          t.notEqual(valid_keys.indexOf(key), -1, key + ' is valid');
+        }
+      }
+      t.end();
+    });
+  }
+  for (keys in schemas) { 
+    isValid(keys, schemas[keys]);
+  }
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('schemas: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
--- a/test/unit/mock/alpha3.json
+++ b/test/unit/mock/alpha3.json
@ -0,0 +1,251 @@
+{
+ "ABW": "Aruba",
+ "AFG": "Afghanistan",
+ "AGO": "Angola",
+ "AIA": "Anguilla",
+ "ALA": "Åland Islands",
+ "ALB": "Albania",
+ "AND": "Andorra",
+ "ARE": "United Arab Emirates",
+ "ARG": "Argentina",
+ "ARM": "Armenia",
+ "ASM": "American Samoa",
+ "ATA": "Antarctica",
+ "ATF": "French Southern Territories",
+ "ATG": "Antigua and Barbuda",
+ "AUS": "Australia",
+ "AUT": "Austria",
+ "AZE": "Azerbaijan",
+ "BDI": "Burundi",
+ "BEL": "Belgium",
+ "BEN": "Benin",
+ "BES": "Bonaire, Sint Eustatius and Saba",
+ "BFA": "Burkina Faso",
+ "BGD": "Bangladesh",
+ "BGR": "Bulgaria",
+ "BHR": "Bahrain",
+ "BHS": "Bahamas",
+ "BIH": "Bosnia and Herzegovina",
+ "BLM": "Saint Barthélemy",
+ "BLR": "Belarus",
+ "BLZ": "Belize",
+ "BMU": "Bermuda",
+ "BOL": "Bolivia, Plurinational State of",
+ "BRA": "Brazil",
+ "BRB": "Barbados",
+ "BRN": "Brunei Darussalam",
+ "BTN": "Bhutan",
+ "BVT": "Bouvet Island",
+ "BWA": "Botswana",
+ "CAF": "Central African Republic",
+ "CAN": "Canada",
+ "CCK": "Cocos (Keeling) Islands",
+ "CHE": "Switzerland",
+ "CHL": "Chile",
+ "CHN": "China",
+ "CIV": "Côte d'Ivoire",
+ "CMR": "Cameroon",
+ "COD": "Congo, the Democratic Republic of the",
+ "COG": "Congo",
+ "COK": "Cook Islands",
+ "COL": "Colombia",
+ "COM": "Comoros",
+ "CPV": "Cabo Verde",
+ "CRI": "Costa Rica",
+ "CUB": "Cuba",
+ "CUW": "Curaçao",
+ "CXR": "Christmas Island",
+ "CYM": "Cayman Islands",
+ "CYP": "Cyprus",
+ "CZE": "Czech Republic",
+ "DEU": "Germany",
+ "DJI": "Djibouti",
+ "DMA": "Dominica",
+ "DNK": "Denmark",
+ "DOM": "Dominican Republic",
+ "DZA": "Algeria",
+ "ECU": "Ecuador",
+ "EGY": "Egypt",
+ "ERI": "Eritrea",
+ "ESH": "Western Sahara",
+ "ESP": "Spain",
+ "EST": "Estonia",
+ "ETH": "Ethiopia",
+ "FIN": "Finland",
+ "FJI": "Fiji",
+ "FLK": "Falkland Islands (Malvinas)",
+ "FRA": "France",
+ "FRO": "Faroe Islands",
+ "FSM": "Micronesia, Federated States of",
+ "GAB": "Gabon",
+ "GBR": "United Kingdom",
+ "GEO": "Georgia",
+ "GGY": "Guernsey",
+ "GHA": "Ghana",
+ "GIB": "Gibraltar",
+ "GIN": "Guinea",
+ "GLP": "Guadeloupe",
+ "GMB": "Gambia",
+ "GNB": "Guinea-Bissau",
+ "GNQ": "Equatorial Guinea",
+ "GRC": "Greece",
+ "GRD": "Grenada",
+ "GRL": "Greenland",
+ "GTM": "Guatemala",
+ "GUF": "French Guiana",
+ "GUM": "Guam",
+ "GUY": "Guyana",
+ "HKG": "Hong Kong",
+ "HMD": "Heard Island and McDonald Islands",
+ "HND": "Honduras",
+ "HRV": "Croatia",
+ "HTI": "Haiti",
+ "HUN": "Hungary",
+ "IDN": "Indonesia",
+ "IMN": "Isle of Man",
+ "IND": "India",
+ "IOT": "British Indian Ocean Territory",
+ "IRL": "Ireland",
+ "IRN": "Iran, Islamic Republic of",
+ "IRQ": "Iraq",
+ "ISL": "Iceland",
+ "ISR": "Israel",
+ "ITA": "Italy",
+ "JAM": "Jamaica",
+ "JEY": "Jersey",
+ "JOR": "Jordan",
+ "JPN": "Japan",
+ "KAZ": "Kazakhstan",
+ "KEN": "Kenya",
+ "KGZ": "Kyrgyzstan",
+ "KHM": "Cambodia",
+ "KIR": "Kiribati",
+ "KNA": "Saint Kitts and Nevis",
+ "KOR": "Korea, Republic of",
+ "KWT": "Kuwait",
+ "LAO": "Lao People's Democratic Republic",
+ "LBN": "Lebanon",
+ "LBR": "Liberia",
+ "LBY": "Libya",
+ "LCA": "Saint Lucia",
+ "LIE": "Liechtenstein",
+ "LKA": "Sri Lanka",
+ "LSO": "Lesotho",
+ "LTU": "Lithuania",
+ "LUX": "Luxembourg",
+ "LVA": "Latvia",
+ "MAC": "Macao",
+ "MAF": "Saint Martin (French part)",
+ "MAR": "Morocco",
+ "MCO": "Monaco",
+ "MDA": "Moldova, Republic of",
+ "MDG": "Madagascar",
+ "MDV": "Maldives",
+ "MEX": "Mexico",
+ "MHL": "Marshall Islands",
+ "MKD": "Macedonia, the former Yugoslav Republic of",
+ "MLI": "Mali",
+ "MLT": "Malta",
+ "MMR": "Myanmar",
+ "MNE": "Montenegro",
+ "MNG": "Mongolia",
+ "MNP": "Northern Mariana Islands",
+ "MOZ": "Mozambique",
+ "MRT": "Mauritania",
+ "MSR": "Montserrat",
+ "MTQ": "Martinique",
+ "MUS": "Mauritius",
+ "MWI": "Malawi",
+ "MYS": "Malaysia",
+ "MYT": "Mayotte",
+ "NAM": "Namibia",
+ "NCL": "New Caledonia",
+ "NER": "Niger",
+ "NFK": "Norfolk Island",
+ "NGA": "Nigeria",
+ "NIC": "Nicaragua",
+ "NIU": "Niue",
+ "NLD": "Netherlands",
+ "NOR": "Norway",
+ "NPL": "Nepal",
+ "NRU": "Nauru",
+ "NZL": "New Zealand",
+ "OMN": "Oman",
+ "PAK": "Pakistan",
+ "PAN": "Panama",
+ "PCN": "Pitcairn",
+ "PER": "Peru",
+ "PHL": "Philippines",
+ "PLW": "Palau",
+ "PNG": "Papua New Guinea",
+ "POL": "Poland",
+ "PRI": "Puerto Rico",
+ "PRK": "Korea, Democratic People's Republic of",
+ "PRT": "Portugal",
+ "PRY": "Paraguay",
+ "PSE": "Palestine, State of",
+ "PYF": "French Polynesia",
+ "QAT": "Qatar",
+ "REU": "Réunion",
+ "ROU": "Romania",
+ "RUS": "Russian Federation",
+ "RWA": "Rwanda",
+ "SAU": "Saudi Arabia",
+ "SDN": "Sudan",
+ "SEN": "Senegal",
+ "SGP": "Singapore",
+ "SGS": "South Georgia and the South Sandwich Islands",
+ "SHN": "Saint Helena, Ascension and Tristan da Cunha",
+ "SJM": "Svalbard and Jan Mayen",
+ "SLB": "Solomon Islands",
+ "SLE": "Sierra Leone",
+ "SLV": "El Salvador",
+ "SMR": "San Marino",
+ "SOM": "Somalia",
+ "SPM": "Saint Pierre and Miquelon",
+ "SRB": "Serbia",
+ "SSD": "South Sudan",
+ "STP": "Sao Tome and Principe",
+ "SUR": "Suriname",
+ "SVK": "Slovakia",
+ "SVN": "Slovenia",
+ "SWE": "Sweden",
+ "SWZ": "Swaziland",
+ "SXM": "Sint Maarten (Dutch part)",
+ "SYC": "Seychelles",
+ "SYR": "Syrian Arab Republic",
+ "TCA": "Turks and Caicos Islands",
+ "TCD": "Chad",
+ "TGO": "Togo",
+ "THA": "Thailand",
+ "TJK": "Tajikistan",
+ "TKL": "Tokelau",
+ "TKM": "Turkmenistan",
+ "TLS": "Timor-Leste",
+ "TON": "Tonga",
+ "TTO": "Trinidad and Tobago",
+ "TUN": "Tunisia",
+ "TUR": "Turkey",
+ "TUV": "Tuvalu",
+ "TWN": "Taiwan, Province of China",
+ "TZA": "Tanzania, United Republic of",
+ "UGA": "Uganda",
+ "UKR": "Ukraine",
+ "UMI": "United States Minor Outlying Islands",
+ "URY": "Uruguay",
+ "USA": "United States",
+ "UZB": "Uzbekistan",
+ "VAT": "Holy See (Vatican City State)",
+ "VCT": "Saint Vincent and the Grenadines",
+ "VEN": "Venezuela, Bolivarian Republic of",
+ "VGB": "Virgin Islands, British",
+ "VIR": "Virgin Islands, U.S.",
+ "VNM": "Viet Nam",
+ "VUT": "Vanuatu",
+ "WLF": "Wallis and Futuna",
+ "WSM": "Samoa",
+ "YEM": "Yemen",
+ "ZAF": "South Africa",
+ "ZMB": "Zambia",
+ "ZWE": "Zimbabwe"
+ }
--- a/test/unit/mock/backend.js
+++ b/test/unit/mock/backend.js
@ -1,20 +1,15 @@

-var mockPayload = function(id){
-  return { 
-    id: 'mocktype/mockid'+id,
-    geo: '101,-10.1'
-  }
-};
-
 var responses = {};
 responses['client/suggest/ok/1'] = function( cmd, cb ){
-  return cb( undefined, suggestEnvelope([ { value: 1, payload: mockPayload(1) }, { value: 2, payload: mockPayload(2) } ]) );
+  return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid1' }, { score: 2, text: 'mocktype:mockid2' } ]) );
 };
 responses['client/suggest/fail/1'] = function( cmd, cb ){
  return cb( 'a backend error occurred' );
 };
 responses['client/search/ok/1'] = function( cmd, cb ){
  return cb( undefined, searchEnvelope([{
+    _id: 'myid1',
+    _type: 'mytype1',
    _source: {
      value: 1,
      center_point: { lat: 100.1, lon: -50.5 },
@ -22,6 +17,8 @@ responses['client/search/ok/1'] = function( cmd, cb ){
      admin0: 'country1', admin1: 'state1', admin2: 'city1'
    }
  }, {
+    _id: 'myid2',
+    _type: 'mytype2',
    _source: {
      value: 2,
      center_point: { lat: 100.2, lon: -51.5 },
@ -34,8 +31,11 @@ responses['client/search/fail/1'] = function( cmd, cb ){
  return cb( 'a backend error occurred' );
 };

-responses['client/doc/ok/1'] = function( cmd, cb ){
-  return cb( undefined, docEnvelope([{
+responses['client/mget/ok/1'] = function( cmd, cb ){
+  return cb( undefined, mgetEnvelope([{
+    _id: 'myid1',
+    _type: 'mytype1',
+    found: true,
    _source: {
      value: 1,
      center_point: { lat: 100.1, lon: -50.5 },
@ -43,6 +43,9 @@ responses['client/doc/ok/1'] = function( cmd, cb ){
      admin0: 'country1', admin1: 'state1', admin2: 'city1'
    }
  }, {
+    _id: 'myid2',
+    _type: 'mytype2',
+    found: true,
    _source: {
      value: 2,
      center_point: { lat: 100.2, lon: -51.5 },
@ -51,7 +54,7 @@ responses['client/doc/ok/1'] = function( cmd, cb ){
    }
  }]));
 };
-responses['client/doc/fail/1'] = responses['client/search/fail/1'];
+responses['client/mget/fail/1'] = responses['client/search/fail/1'];

 function setup( key, cmdCb ){
  function backend( a, b ){
@ -59,7 +62,7 @@ function setup( key, cmdCb ){
      client: {
        mget: function( cmd, cb ){
          if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
-          return responses[key].apply( this, arguments );
+          return responses[key.indexOf('mget') === -1 ? 'client/mget/ok/1' : key].apply( this, arguments );
        },
        suggest: function( cmd, cb ){
          if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
@ -75,7 +78,7 @@ function setup( key, cmdCb ){
  return backend;
 }

-function docEnvelope( options ){
+function mgetEnvelope( options ){
  return { docs: options };
 }

--- a/test/unit/run.js
+++ b/test/unit/run.js
@ -17,7 +17,8 @@ var tests = [
  require('./query/suggest'),
  require('./query/search'),
  require('./query/reverse'),
-  require('./helper/geojsonify')
+  require('./helper/geojsonify'),
+  require('./helper/outputSchema')
 ];

 tests.map(function(t) {
--- a/test/unit/service/mget.js
+++ b/test/unit/service/mget.js
@ -16,12 +16,14 @@ module.exports.tests.functional_success = function(test, common) {

  var expected = [
    {
+      _id: 'myid1', _type: 'mytype1',
      value: 1,
      center_point: { lat: 100.1, lon: -50.5 },
      name: { default: 'test name1' },
      admin0: 'country1', admin1: 'state1', admin2: 'city1'
    }, 
    {
+      _id: 'myid2', _type: 'mytype2',
      value: 2,
      center_point: { lat: 100.2, lon: -51.5 },
      name: { default: 'test name2' },
@ -30,7 +32,7 @@ module.exports.tests.functional_success = function(test, common) {
  ];

  test('valid query', function(t) {
-    var backend = mockBackend( 'client/doc/ok/1', function( cmd ){
+    var backend = mockBackend( 'client/mget/ok/1', function( cmd ){
      t.deepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'correct backend command');
    });
    setup( backend, [ { _id: 123, _index: 'pelias', _type: 'a' } ], function(err, data) {
@ -56,7 +58,7 @@ module.exports.tests.functional_failure = function(test, common) {
      { }
    ];

-    var backend = mockBackend( 'client/doc/fail/1', function( cmd ){
+    var backend = mockBackend( 'client/mget/fail/1', function( cmd ){
      t.notDeepEqual(cmd, { body: { docs: [ { _id: 123, _index: 'pelias', _type: 'a' } ] } }, 'incorrect backend command');
    });
    invalid_queries.forEach(function(query) {
--- a/test/unit/service/search.js
+++ b/test/unit/service/search.js
@ -18,12 +18,14 @@ module.exports.tests.functional_success = function(test, common) {

  var expected = [
    {
+      _id: 'myid1', _type: 'mytype1',
      value: 1,
      center_point: { lat: 100.1, lon: -50.5 },
      name: { default: 'test name1' },
      admin0: 'country1', admin1: 'state1', admin2: 'city1'
    }, 
    {
+      _id: 'myid2', _type: 'mytype2',
      value: 2,
      center_point: { lat: 100.2, lon: -51.5 },
      name: { default: 'test name2' },
--- a/test/unit/service/suggest.js
+++ b/test/unit/service/suggest.js
@ -16,16 +16,9 @@ module.exports.tests.interface = function(test, common) {
 // functionally test service 
 module.exports.tests.functional_success = function(test, common) {

-  var mockPayload = function(id){
-    return { 
-      id: 'mocktype/mockid'+id,
-      geo: '101,-10.1'
-    }
-  };
-
  var expected = [ 
-    { value: 1, payload: mockPayload(1) }, 
-    { value: 2, payload: mockPayload(2) } 
+    { score: 1, text: 'mocktype:mockid1' }, 
+    { score: 2, text: 'mocktype:mockid2' } 
  ];

  test('valid ES query', function(t) {