From 6ddbe86ac5634553340bc5708249e6f3f1a3fc41 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Mon, 3 Nov 2014 17:30:21 +0000
Subject: [PATCH 1/5] perform mget during suggest. requires suggester output to
 be a gid

---
 controller/suggest.js            | 48 ++++++++++++++++++++++++++------
 test/ciao/suggest/success.coffee |  1 +
 test/unit/controller/suggest.js  | 31 ++++++++++++++-------
 test/unit/mock/backend.js        | 31 +++++++++++++++++----
 4 files changed, 88 insertions(+), 23 deletions(-)

diff --git a/controller/suggest.js b/controller/suggest.js
index 956363b8..b0a30f70 100644
--- a/controller/suggest.js
+++ b/controller/suggest.js
@@ -1,6 +1,9 @@
 
-var service = { suggest: require('../service/suggest') };
-var geojsonify = require('../helper/geojsonify').suggest;
+var service = {
+  suggest: require('../service/suggest'),
+  mget: require('../service/mget')
+};
+var geojsonify = require('../helper/geojsonify').search;
 
 function setup( backend, query ){
 
@@ -16,12 +19,9 @@ function setup( backend, query ){
       body: query( req.clean )
     };
 
-    // query backend
-    service.suggest( backend, cmd, function( err, docs ){
-
-      // error handler
-      if( err ){ return next( err ); }
-
+    // responder
+    function reply( docs ){
+      
       // convert docs to geojson
       var geojson = geojsonify( docs );
 
@@ -30,6 +30,38 @@ function setup( backend, query ){
 
       // respond
       return res.status(200).json( geojson );
+    }
+
+    // query backend
+    service.suggest( backend, cmd, function( err, suggested ){
+
+      // error handler
+      if( err ){ return next( err ); }
+
+      // no documents suggested, return empty array to avoid ActionRequestValidationException
+      if( !Array.isArray( suggested ) || !suggested.length ){
+        return reply([]);
+      }
+
+      // map suggester output to mget query
+      var query = suggested.map( function( doc ) {
+        var idParts = doc.text.split(':');
+        return {
+          _index: 'pelias',
+          _type: idParts[0],
+          _id: idParts[1]
+        };
+      });
+
+      service.mget( backend, query, function( err, docs ){
+
+        // error handler
+        if( err ){ return next( err ); }
+
+        // reply
+        return reply( docs );
+
+      });
     });
 
   }
diff --git a/test/ciao/suggest/success.coffee b/test/ciao/suggest/success.coffee
index e54e6424..08e504d2 100644
--- a/test/ciao/suggest/success.coffee
+++ b/test/ciao/suggest/success.coffee
@@ -12,5 +12,6 @@ should.not.exist json.error
 json.date.should.be.within now-5000, now+5000
 
 #? valid geojson
+console.log( json );
 json.type.should.equal 'FeatureCollection'
 json.features.should.be.instanceof Array
\ No newline at end of file
diff --git a/test/unit/controller/suggest.js b/test/unit/controller/suggest.js
index f67f5da7..1553d348 100644
--- a/test/unit/controller/suggest.js
+++ b/test/unit/controller/suggest.js
@@ -16,34 +16,42 @@ module.exports.tests.interface = function(test, common) {
 // functionally test controller (backend success)
 module.exports.tests.functional_success = function(test, common) {
 
-  // expected geojson features for 'client/suggest/ok/1' fixture
+  // expected geojson features for 'client/mget/ok/1' fixture
   var expected = [{
     type: 'Feature',
     geometry: {
       type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -50.5, 100.1 ]
     },
     properties: {
-      id: 'mockid',
-      type: 'mocktype',
-      value: 1
+      name: 'test name1',
+      admin0: 'country1',
+      admin1: 'state1',
+      admin2: 'city1'
     }
   }, {
     type: 'Feature',
     geometry: {
       type: 'Point',
-      coordinates: [ 101, -10.1 ]
+      coordinates: [ -51.5, 100.2 ]
     },
     properties: {
-      id: 'mockid',
-      type: 'mocktype',
-      value: 2
+      name: 'test name2',
+      admin0: 'country2',
+      admin1: 'state2',
+      admin2: 'city2'
     }
   }];
 
   test('functional success', function(t) {
+    var i = 0;
     var backend = mockBackend( 'client/suggest/ok/1', function( cmd ){
-      t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct backend command');
+      // the backend executes 2 commands, so we check them both
+      if( ++i === 1 ){
+        t.deepEqual(cmd, { body: { a: 'b' }, index: 'pelias' }, 'correct suggest command');
+      } else {
+        t.deepEqual(cmd, { body: { docs: [ { _id: 'mockid', _index: 'pelias', _type: 'mocktype' }, { _id: 'mockid', _index: 'pelias', _type: 'mocktype' } ] } }, 'correct mget command');
+      }
     });
     var controller = setup( backend, mockQuery() );
     var res = {
@@ -52,6 +60,9 @@ module.exports.tests.functional_success = function(test, common) {
         return res;
       },
       json: function( json ){
+
+        console.log( 'json', json );
+
         t.equal(typeof json, 'object', 'returns json');
         t.equal(typeof json.date, 'number', 'date set');
         t.equal(json.type, 'FeatureCollection', 'valid geojson');
diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js
index 63378423..f2759168 100644
--- a/test/unit/mock/backend.js
+++ b/test/unit/mock/backend.js
@@ -1,12 +1,8 @@
 
-var mockPayload = {
-  id: 'mocktype/mockid',
-  geo: '101,-10.1'
-};
 
 var responses = {};
 responses['client/suggest/ok/1'] = function( cmd, cb ){
-  return cb( undefined, suggestEnvelope([ { value: 1, payload: mockPayload }, { value: 2, payload: mockPayload } ]) );
+  return cb( undefined, suggestEnvelope([ { score: 1, text: 'mocktype:mockid' }, { score: 2, text: 'mocktype:mockid' } ]) );
 };
 responses['client/suggest/fail/1'] = function( cmd, cb ){
   return cb( 'a backend error occurred' );
@@ -28,6 +24,23 @@ responses['client/search/ok/1'] = function( cmd, cb ){
     }
   }]));
 };
+responses['client/mget/ok/1'] = function( cmd, cb ){
+  return cb( undefined, mgetEnvelope([{
+    _source: {
+      value: 1,
+      center_point: { lat: 100.1, lon: -50.5 },
+      name: { default: 'test name1' },
+      admin0: 'country1', admin1: 'state1', admin2: 'city1'
+    }
+  }, {
+    _source: {
+      value: 2,
+      center_point: { lat: 100.2, lon: -51.5 },
+      name: { default: 'test name2' },
+      admin0: 'country2', admin1: 'state2', admin2: 'city2'
+    }
+  }]));
+};
 responses['client/search/fail/1'] = function( cmd, cb ){
   return cb( 'a backend error occurred' );
 };
@@ -43,6 +56,10 @@ function setup( key, cmdCb ){
         search: function( cmd, cb ){
           if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
           return responses[key].apply( this, arguments );
+        },
+        mget: function( cmd, cb ){
+          if( 'function' === typeof cmdCb ){ cmdCb( cmd ); }
+          return responses['client/mget/ok/1'].apply( this, arguments );
         }
       }
     };
@@ -58,4 +75,8 @@ function searchEnvelope( options ){
   return { hits: { total: options.length, hits: options } };
 }
 
+function mgetEnvelope( options ){
+  return { docs: options };
+}
+
 module.exports = setup;
\ No newline at end of file

From 88aa48ec016ed36b37a8f9cc40613ba5ef98c6b3 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Mon, 3 Nov 2014 17:33:57 +0000
Subject: [PATCH 2/5] formatting

---
 test/ciao/suggest/success.coffee | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/ciao/suggest/success.coffee b/test/ciao/suggest/success.coffee
index 08e504d2..e54e6424 100644
--- a/test/ciao/suggest/success.coffee
+++ b/test/ciao/suggest/success.coffee
@@ -12,6 +12,5 @@ should.not.exist json.error
 json.date.should.be.within now-5000, now+5000
 
 #? valid geojson
-console.log( json );
 json.type.should.equal 'FeatureCollection'
 json.features.should.be.instanceof Array
\ No newline at end of file

From fcd60463cca0cf61137d0e2e130f6a8cf72b923e Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Mon, 3 Nov 2014 17:41:37 +0000
Subject: [PATCH 3/5] remove geojsonify.suggest

---
 helper/geojsonify.js           | 42 ----------------------
 test/unit/helper/geojsonify.js | 66 ++--------------------------------
 2 files changed, 3 insertions(+), 105 deletions(-)

diff --git a/helper/geojsonify.js b/helper/geojsonify.js
index d4263900..196bd18f 100644
--- a/helper/geojsonify.js
+++ b/helper/geojsonify.js
@@ -1,47 +1,6 @@
 
 var GeoJSON = require('geojson');
 
-function suggest( docs ){
-
-  // emit a warning if the doc format is invalid
-  // @note: if you see this error, fix it ASAP!
-  function warning(){
-    console.error( 'error: invalid doc', __filename );
-    return false; // remove offending doc from results
-  }
-
-  // flatten & expand data for geojson conversion
-  var geodata = docs.map( function( doc ){
-
-    // something went very wrong
-    if( !doc || !doc.payload ) return warning();
-
-    // split payload id string in to geojson properties
-    if( 'string' !== typeof doc.payload.id ) return warning();
-    var idParts = doc.payload.id.split('/');
-    doc.type = idParts[0];
-    doc.id = idParts[1];
-
-    // split payload geo string in to geojson properties
-    if( 'string' !== typeof doc.payload.geo ) return warning();
-    var geoParts = doc.payload.geo.split(',');
-    doc.lat = parseFloat( geoParts[1] );
-    doc.lng = parseFloat( geoParts[0] );
-
-    // remove payload from doc
-    delete doc.payload;
-    return doc;
-
-  // filter-out invalid entries
-  }).filter( function( doc ){
-    return doc;
-  });
-
-  // convert to geojson
-  return GeoJSON.parse( geodata, { Point: ['lat', 'lng'] } );
-
-}
-
 function search( docs ){
 
   // emit a warning if the doc format is invalid
@@ -95,5 +54,4 @@ function search( docs ){
 
 }
 
-module.exports.suggest = suggest;
 module.exports.search = search;
\ No newline at end of file
diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js
index f6bdff2b..d855e390 100644
--- a/test/unit/helper/geojsonify.js
+++ b/test/unit/helper/geojsonify.js
@@ -4,73 +4,13 @@ var geojsonify = require('../../../helper/geojsonify');
 module.exports.tests = {};
 
 module.exports.tests.interface = function(test, common) {
-  test('valid interface .suggest()', function(t) {
-    t.equal(typeof geojsonify.suggest, 'function', 'suggest is a function');
-    t.equal(geojsonify.suggest.length, 1, 'accepts x arguments');
+  test('valid interface .search()', function(t) {
+    t.equal(typeof geojsonify.search, 'function', 'search is a function');
+    t.equal(geojsonify.search.length, 1, 'accepts x arguments');
     t.end();
   });
 };
 
-module.exports.tests.suggest = function(test, common) {
-
-  var input = [{
-    bingo1: 'bango1',
-    payload: {
-      id: 'foo1/bar1',
-      geo: '100,-10.5'
-    }
-  },{
-    bingo2: 'bango2',
-    payload: {
-      id: 'foo2/bar2',
-      geo: '10,-1.5'
-    }
-  }];
-
-  var expected = {
-    "type": "FeatureCollection",
-    "features": [
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            100,
-            -10.5
-          ]
-        },
-        "properties": {
-          "bingo1": "bango1",
-          "type": "foo1",
-          "id": "bar1"
-        }
-      },
-      {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            10,
-            -1.5
-          ]
-        },
-        "properties": {
-          "bingo2": "bango2",
-          "type": "foo2",
-          "id": "bar2"
-        }
-      }
-    ]
-  };
-
-  test('geojsonify.suggest()', function(t) {
-    var json = geojsonify.suggest( input );
-    t.deepEqual(json, expected, 'all docs mapped');
-    t.end();
-  });
-};
-
-
 module.exports.tests.search = function(test, common) {
 
   var input = [

From 27e2c4e5b8bf0eebf2aebbcad9d4eb5405464a16 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Mon, 3 Nov 2014 17:54:22 +0000
Subject: [PATCH 4/5] migrate outputGenerator from pelias/suggester-pipeline

---
 helper/geojsonify.js             |   9 +-
 helper/outputGenerator.js        |  37 +++++
 helper/outputSchema.json         |  14 ++
 test/unit/controller/search.js   |   6 +-
 test/unit/controller/suggest.js  |   6 +-
 test/unit/helper/geojsonify.js   |   2 +-
 test/unit/helper/outputSchema.js |  55 +++++++
 test/unit/mock/alpha3.json       | 251 +++++++++++++++++++++++++++++++
 test/unit/run.js                 |   3 +-
 9 files changed, 372 insertions(+), 11 deletions(-)
 create mode 100644 helper/outputGenerator.js
 create mode 100644 helper/outputSchema.json
 create mode 100644 test/unit/helper/outputSchema.js
 create mode 100644 test/unit/mock/alpha3.json

diff --git a/helper/geojsonify.js b/helper/geojsonify.js
index 196bd18f..7bdae990 100644
--- a/helper/geojsonify.js
+++ b/helper/geojsonify.js
@@ -1,5 +1,6 @@
 
-var GeoJSON = require('geojson');
+var GeoJSON = require('geojson'),
+    outputGenerator = require('./outputGenerator');
 
 function search( docs ){
 
@@ -37,10 +38,8 @@ function search( docs ){
     if( doc.locality ){ output.locality = doc.locality; }
     if( doc.neighborhood ){ output.neighborhood = doc.neighborhood; }
 
-    // map suggest output
-    if( doc.suggest && doc.suggest.output ){
-      output.text = doc.suggest.output;
-    }
+    // generate region-specific text string
+    output.text = outputGenerator( doc );
 
     return output;
 
diff --git a/helper/outputGenerator.js b/helper/outputGenerator.js
new file mode 100644
index 00000000..42fa1279
--- /dev/null
+++ b/helper/outputGenerator.js
@@ -0,0 +1,37 @@
+
+var schemas = require('./outputSchema.json');
+
+module.exports = function( record ){
+
+  var adminParts = [];
+
+  var schema = schemas.default;
+  
+  if (record.alpha3 && record.alpha3.length && schemas[record.alpha3]) {
+    schema = schemas[record.alpha3];
+  }
+  
+  var buildOutput = function(parts, schemaArr, record) {
+    for (var i=0; i<schemaArr.length; i++) {
+      var rec = record[schemaArr[i]];
+      if (rec && rec.length) {
+        parts.push( rec );
+        return parts;
+      }
+    }
+    return parts;
+  };
+
+  for (var key in schema) {
+    adminParts = buildOutput(adminParts, schema[key], record);  
+  }
+
+  var outputs = [ record.name.default ].concat( adminParts );
+
+  // de-dupe outputs
+  outputs = outputs.filter( function( i, pos ) {
+    return outputs.indexOf( i ) == pos;
+  });
+
+  return outputs.join(', ').trim();
+};
\ No newline at end of file
diff --git a/helper/outputSchema.json b/helper/outputSchema.json
new file mode 100644
index 00000000..430b8e1a
--- /dev/null
+++ b/helper/outputSchema.json
@@ -0,0 +1,14 @@
+{
+  "USA": {
+    "local": ["local_admin", "locality", "neighborhood", "admin2"],
+    "regional": ["admin1_abbr", "admin1", "admin0"]
+  },
+  "GBR": {
+    "local": ["admin2","neighborhood", "local_admin", "locality"],
+    "regional": ["admin0","admin1"]
+  },
+  "default": {
+    "local": ["local_admin", "locality", "neighborhood", "admin2"],
+    "regional": ["admin1_abbr", "admin1", "admin0"]
+  }
+}
\ No newline at end of file
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 72dadada..7e714118 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -27,7 +27,8 @@ module.exports.tests.functional_success = function(test, common) {
       name: 'test name1',
       admin0: 'country1',
       admin1: 'state1',
-      admin2: 'city1'
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
     }
   }, {
     type: 'Feature',
@@ -39,7 +40,8 @@ module.exports.tests.functional_success = function(test, common) {
       name: 'test name2',
       admin0: 'country2',
       admin1: 'state2',
-      admin2: 'city2'
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
     }
   }];
 
diff --git a/test/unit/controller/suggest.js b/test/unit/controller/suggest.js
index 1553d348..80eb9eea 100644
--- a/test/unit/controller/suggest.js
+++ b/test/unit/controller/suggest.js
@@ -27,7 +27,8 @@ module.exports.tests.functional_success = function(test, common) {
       name: 'test name1',
       admin0: 'country1',
       admin1: 'state1',
-      admin2: 'city1'
+      admin2: 'city1',
+      text: 'test name1, city1, state1'
     }
   }, {
     type: 'Feature',
@@ -39,7 +40,8 @@ module.exports.tests.functional_success = function(test, common) {
       name: 'test name2',
       admin0: 'country2',
       admin1: 'state2',
-      admin2: 'city2'
+      admin2: 'city2',
+      text: 'test name2, city2, state2'
     }
   }];
 
diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js
index d855e390..61a71eeb 100644
--- a/test/unit/helper/geojsonify.js
+++ b/test/unit/helper/geojsonify.js
@@ -72,7 +72,7 @@ module.exports.tests.search = function(test, common) {
           "id": "osmway/147495160",
           "geo": "-0.101795,51.517806"
         },
-        "output": "Blues Cafe, Smithfield, United Kingdom"
+        "output": "Blues Cafe, test3, COL"
       }
     }
   ];
diff --git a/test/unit/helper/outputSchema.js b/test/unit/helper/outputSchema.js
new file mode 100644
index 00000000..0bab157a
--- /dev/null
+++ b/test/unit/helper/outputSchema.js
@@ -0,0 +1,55 @@
+
+var schemas = require('../../../helper/outputSchema.json');
+var alpha3  = require('../mock/alpha3.json');
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('interface', function(t) {
+    t.equal(typeof schemas, 'object', 'valid object');
+    t.equal(schemas.hasOwnProperty('default'), true, 'has default defined');
+    t.end();
+  });
+};
+
+module.exports.tests.valid = function(test, common) {
+  var valid_keys = ['local_admin', 'locality', 'neighborhood', 'admin2', 'admin1_abbr', 'admin1', 'admin0'];
+  var default_schema = {
+    local: ['local_admin', 'locality', 'neighborhood', 'admin2'],
+    regional: ['admin1_abbr', 'admin1', 'admin0']
+  };
+
+  var isValid = function(keys, schema) {
+    test('valid key/object (' + keys + ')' , function(t) {
+      if (keys=="default") { 
+        t.deepEqual(schema, default_schema, 'valid default schema');
+      } else {
+        t.equal(alpha3.hasOwnProperty(keys), true, 'valid key');  
+      }
+      t.equal(typeof schema, 'object', 'valid object');
+      t.notEqual(Object.getOwnPropertyNames(schema).length, 0,  'object not empty');
+      for (levels in schema) {
+        t.equal(Object.prototype.toString.call(schema[levels]), '[object Array]', levels+' is an array');
+        for (var i=0;i<schema[levels].length;i++) {
+          var key = schema[levels][i];
+          t.notEqual(valid_keys.indexOf(key), -1, key + ' is valid');
+        }
+      }
+      t.end();
+    });
+  }
+  for (keys in schemas) { 
+    isValid(keys, schemas[keys]);
+  }
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('schemas: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};
\ No newline at end of file
diff --git a/test/unit/mock/alpha3.json b/test/unit/mock/alpha3.json
new file mode 100644
index 00000000..7148e878
--- /dev/null
+++ b/test/unit/mock/alpha3.json
@@ -0,0 +1,251 @@
+{
+ "ABW": "Aruba",
+ "AFG": "Afghanistan",
+ "AGO": "Angola",
+ "AIA": "Anguilla",
+ "ALA": "Åland Islands",
+ "ALB": "Albania",
+ "AND": "Andorra",
+ "ARE": "United Arab Emirates",
+ "ARG": "Argentina",
+ "ARM": "Armenia",
+ "ASM": "American Samoa",
+ "ATA": "Antarctica",
+ "ATF": "French Southern Territories",
+ "ATG": "Antigua and Barbuda",
+ "AUS": "Australia",
+ "AUT": "Austria",
+ "AZE": "Azerbaijan",
+ "BDI": "Burundi",
+ "BEL": "Belgium",
+ "BEN": "Benin",
+ "BES": "Bonaire, Sint Eustatius and Saba",
+ "BFA": "Burkina Faso",
+ "BGD": "Bangladesh",
+ "BGR": "Bulgaria",
+ "BHR": "Bahrain",
+ "BHS": "Bahamas",
+ "BIH": "Bosnia and Herzegovina",
+ "BLM": "Saint Barthélemy",
+ "BLR": "Belarus",
+ "BLZ": "Belize",
+ "BMU": "Bermuda",
+ "BOL": "Bolivia, Plurinational State of",
+ "BRA": "Brazil",
+ "BRB": "Barbados",
+ "BRN": "Brunei Darussalam",
+ "BTN": "Bhutan",
+ "BVT": "Bouvet Island",
+ "BWA": "Botswana",
+ "CAF": "Central African Republic",
+ "CAN": "Canada",
+ "CCK": "Cocos (Keeling) Islands",
+ "CHE": "Switzerland",
+ "CHL": "Chile",
+ "CHN": "China",
+ "CIV": "Côte d'Ivoire",
+ "CMR": "Cameroon",
+ "COD": "Congo, the Democratic Republic of the",
+ "COG": "Congo",
+ "COK": "Cook Islands",
+ "COL": "Colombia",
+ "COM": "Comoros",
+ "CPV": "Cabo Verde",
+ "CRI": "Costa Rica",
+ "CUB": "Cuba",
+ "CUW": "Curaçao",
+ "CXR": "Christmas Island",
+ "CYM": "Cayman Islands",
+ "CYP": "Cyprus",
+ "CZE": "Czech Republic",
+ "DEU": "Germany",
+ "DJI": "Djibouti",
+ "DMA": "Dominica",
+ "DNK": "Denmark",
+ "DOM": "Dominican Republic",
+ "DZA": "Algeria",
+ "ECU": "Ecuador",
+ "EGY": "Egypt",
+ "ERI": "Eritrea",
+ "ESH": "Western Sahara",
+ "ESP": "Spain",
+ "EST": "Estonia",
+ "ETH": "Ethiopia",
+ "FIN": "Finland",
+ "FJI": "Fiji",
+ "FLK": "Falkland Islands (Malvinas)",
+ "FRA": "France",
+ "FRO": "Faroe Islands",
+ "FSM": "Micronesia, Federated States of",
+ "GAB": "Gabon",
+ "GBR": "United Kingdom",
+ "GEO": "Georgia",
+ "GGY": "Guernsey",
+ "GHA": "Ghana",
+ "GIB": "Gibraltar",
+ "GIN": "Guinea",
+ "GLP": "Guadeloupe",
+ "GMB": "Gambia",
+ "GNB": "Guinea-Bissau",
+ "GNQ": "Equatorial Guinea",
+ "GRC": "Greece",
+ "GRD": "Grenada",
+ "GRL": "Greenland",
+ "GTM": "Guatemala",
+ "GUF": "French Guiana",
+ "GUM": "Guam",
+ "GUY": "Guyana",
+ "HKG": "Hong Kong",
+ "HMD": "Heard Island and McDonald Islands",
+ "HND": "Honduras",
+ "HRV": "Croatia",
+ "HTI": "Haiti",
+ "HUN": "Hungary",
+ "IDN": "Indonesia",
+ "IMN": "Isle of Man",
+ "IND": "India",
+ "IOT": "British Indian Ocean Territory",
+ "IRL": "Ireland",
+ "IRN": "Iran, Islamic Republic of",
+ "IRQ": "Iraq",
+ "ISL": "Iceland",
+ "ISR": "Israel",
+ "ITA": "Italy",
+ "JAM": "Jamaica",
+ "JEY": "Jersey",
+ "JOR": "Jordan",
+ "JPN": "Japan",
+ "KAZ": "Kazakhstan",
+ "KEN": "Kenya",
+ "KGZ": "Kyrgyzstan",
+ "KHM": "Cambodia",
+ "KIR": "Kiribati",
+ "KNA": "Saint Kitts and Nevis",
+ "KOR": "Korea, Republic of",
+ "KWT": "Kuwait",
+ "LAO": "Lao People's Democratic Republic",
+ "LBN": "Lebanon",
+ "LBR": "Liberia",
+ "LBY": "Libya",
+ "LCA": "Saint Lucia",
+ "LIE": "Liechtenstein",
+ "LKA": "Sri Lanka",
+ "LSO": "Lesotho",
+ "LTU": "Lithuania",
+ "LUX": "Luxembourg",
+ "LVA": "Latvia",
+ "MAC": "Macao",
+ "MAF": "Saint Martin (French part)",
+ "MAR": "Morocco",
+ "MCO": "Monaco",
+ "MDA": "Moldova, Republic of",
+ "MDG": "Madagascar",
+ "MDV": "Maldives",
+ "MEX": "Mexico",
+ "MHL": "Marshall Islands",
+ "MKD": "Macedonia, the former Yugoslav Republic of",
+ "MLI": "Mali",
+ "MLT": "Malta",
+ "MMR": "Myanmar",
+ "MNE": "Montenegro",
+ "MNG": "Mongolia",
+ "MNP": "Northern Mariana Islands",
+ "MOZ": "Mozambique",
+ "MRT": "Mauritania",
+ "MSR": "Montserrat",
+ "MTQ": "Martinique",
+ "MUS": "Mauritius",
+ "MWI": "Malawi",
+ "MYS": "Malaysia",
+ "MYT": "Mayotte",
+ "NAM": "Namibia",
+ "NCL": "New Caledonia",
+ "NER": "Niger",
+ "NFK": "Norfolk Island",
+ "NGA": "Nigeria",
+ "NIC": "Nicaragua",
+ "NIU": "Niue",
+ "NLD": "Netherlands",
+ "NOR": "Norway",
+ "NPL": "Nepal",
+ "NRU": "Nauru",
+ "NZL": "New Zealand",
+ "OMN": "Oman",
+ "PAK": "Pakistan",
+ "PAN": "Panama",
+ "PCN": "Pitcairn",
+ "PER": "Peru",
+ "PHL": "Philippines",
+ "PLW": "Palau",
+ "PNG": "Papua New Guinea",
+ "POL": "Poland",
+ "PRI": "Puerto Rico",
+ "PRK": "Korea, Democratic People's Republic of",
+ "PRT": "Portugal",
+ "PRY": "Paraguay",
+ "PSE": "Palestine, State of",
+ "PYF": "French Polynesia",
+ "QAT": "Qatar",
+ "REU": "Réunion",
+ "ROU": "Romania",
+ "RUS": "Russian Federation",
+ "RWA": "Rwanda",
+ "SAU": "Saudi Arabia",
+ "SDN": "Sudan",
+ "SEN": "Senegal",
+ "SGP": "Singapore",
+ "SGS": "South Georgia and the South Sandwich Islands",
+ "SHN": "Saint Helena, Ascension and Tristan da Cunha",
+ "SJM": "Svalbard and Jan Mayen",
+ "SLB": "Solomon Islands",
+ "SLE": "Sierra Leone",
+ "SLV": "El Salvador",
+ "SMR": "San Marino",
+ "SOM": "Somalia",
+ "SPM": "Saint Pierre and Miquelon",
+ "SRB": "Serbia",
+ "SSD": "South Sudan",
+ "STP": "Sao Tome and Principe",
+ "SUR": "Suriname",
+ "SVK": "Slovakia",
+ "SVN": "Slovenia",
+ "SWE": "Sweden",
+ "SWZ": "Swaziland",
+ "SXM": "Sint Maarten (Dutch part)",
+ "SYC": "Seychelles",
+ "SYR": "Syrian Arab Republic",
+ "TCA": "Turks and Caicos Islands",
+ "TCD": "Chad",
+ "TGO": "Togo",
+ "THA": "Thailand",
+ "TJK": "Tajikistan",
+ "TKL": "Tokelau",
+ "TKM": "Turkmenistan",
+ "TLS": "Timor-Leste",
+ "TON": "Tonga",
+ "TTO": "Trinidad and Tobago",
+ "TUN": "Tunisia",
+ "TUR": "Turkey",
+ "TUV": "Tuvalu",
+ "TWN": "Taiwan, Province of China",
+ "TZA": "Tanzania, United Republic of",
+ "UGA": "Uganda",
+ "UKR": "Ukraine",
+ "UMI": "United States Minor Outlying Islands",
+ "URY": "Uruguay",
+ "USA": "United States",
+ "UZB": "Uzbekistan",
+ "VAT": "Holy See (Vatican City State)",
+ "VCT": "Saint Vincent and the Grenadines",
+ "VEN": "Venezuela, Bolivarian Republic of",
+ "VGB": "Virgin Islands, British",
+ "VIR": "Virgin Islands, U.S.",
+ "VNM": "Viet Nam",
+ "VUT": "Vanuatu",
+ "WLF": "Wallis and Futuna",
+ "WSM": "Samoa",
+ "YEM": "Yemen",
+ "ZAF": "South Africa",
+ "ZMB": "Zambia",
+ "ZWE": "Zimbabwe"
+ }
\ No newline at end of file
diff --git a/test/unit/run.js b/test/unit/run.js
index bad988db..8edf342b 100644
--- a/test/unit/run.js
+++ b/test/unit/run.js
@@ -12,7 +12,8 @@ var tests = [
   require('./query/suggest'),
   require('./query/search'),
   require('./query/reverse'),
-  require('./helper/geojsonify')
+  require('./helper/geojsonify'),
+  require('./helper/outputSchema')
 ];
 
 tests.map(function(t) {

From 6efa0aa510f2b519090bb4d6609ba37d3ca41edc Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Mon, 3 Nov 2014 18:21:27 +0000
Subject: [PATCH 5/5] expose id/type/layer

---
 helper/geojsonify.js            |  9 +++++++--
 service/mget.js                 | 14 +++++++++++++-
 service/search.js               |  6 ++++++
 test/unit/controller/search.js  |  6 ++++++
 test/unit/controller/suggest.js |  6 ++++++
 test/unit/helper/geojsonify.js  | 10 ++++++++++
 test/unit/mock/backend.js       | 10 ++++++++++
 7 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/helper/geojsonify.js b/helper/geojsonify.js
index 7bdae990..630f12be 100644
--- a/helper/geojsonify.js
+++ b/helper/geojsonify.js
@@ -14,11 +14,16 @@ function search( docs ){
   // flatten & expand data for geojson conversion
   var geodata = docs.map( function( doc ){
 
-    var output = {};
-
     // something went very wrong
     if( !doc ) return warning();
 
+    var output = {};
+
+    // provide metadata to consumer
+    output.id = doc._id;
+    output.type = doc._type;
+    output.layer = doc._type;
+
     // map center_point
     if( !doc.center_point ) return warning();
     output.lat = parseFloat( doc.center_point.lat );
diff --git a/service/mget.js b/service/mget.js
index a5713f03..cfc79ca3 100644
--- a/service/mget.js
+++ b/service/mget.js
@@ -29,7 +29,19 @@ function service( backend, query, cb ){
     // map returned documents
     var docs = [];
     if( data && Array.isArray(data.docs) ){
-      docs = data.docs.map( function( doc ){
+
+      docs = data.docs.filter( function( doc ){
+
+        // remove docs not actually found
+        return doc.found;
+      
+      }).map( function( doc ){
+
+        // map metadata in to _source so we
+        // can serve it up to the consumer
+        doc._source._id = doc._id;
+        doc._source._type = doc._type;
+
         return doc._source;
       });
     }
diff --git a/service/search.js b/service/search.js
index 6f525123..efe10cea 100644
--- a/service/search.js
+++ b/service/search.js
@@ -17,6 +17,12 @@ function service( backend, cmd, cb ){
     var docs = [];
     if( data && data.hits && data.hits.total && Array.isArray(data.hits.hits)){
       docs = data.hits.hits.map( function( hit ){
+
+        // map metadata in to _source so we
+        // can serve it up to the consumer
+        hit._source._id = hit._id;
+        hit._source._type = hit._type;
+
         return hit._source;
       });
     }
diff --git a/test/unit/controller/search.js b/test/unit/controller/search.js
index 7e714118..19482eb9 100644
--- a/test/unit/controller/search.js
+++ b/test/unit/controller/search.js
@@ -24,6 +24,9 @@ module.exports.tests.functional_success = function(test, common) {
       coordinates: [ -50.5, 100.1 ]
     },
     properties: {
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
       name: 'test name1',
       admin0: 'country1',
       admin1: 'state1',
@@ -37,6 +40,9 @@ module.exports.tests.functional_success = function(test, common) {
       coordinates: [ -51.5, 100.2 ]
     },
     properties: {
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
       name: 'test name2',
       admin0: 'country2',
       admin1: 'state2',
diff --git a/test/unit/controller/suggest.js b/test/unit/controller/suggest.js
index 80eb9eea..283e78cc 100644
--- a/test/unit/controller/suggest.js
+++ b/test/unit/controller/suggest.js
@@ -24,6 +24,9 @@ module.exports.tests.functional_success = function(test, common) {
       coordinates: [ -50.5, 100.1 ]
     },
     properties: {
+      id: 'myid1',
+      type: 'mytype1',
+      layer: 'mytype1',
       name: 'test name1',
       admin0: 'country1',
       admin1: 'state1',
@@ -37,6 +40,9 @@ module.exports.tests.functional_success = function(test, common) {
       coordinates: [ -51.5, 100.2 ]
     },
     properties: {
+      id: 'myid2',
+      type: 'mytype2',
+      layer: 'mytype2',
       name: 'test name2',
       admin0: 'country2',
       admin1: 'state2',
diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js
index 61a71eeb..27bb8b31 100644
--- a/test/unit/helper/geojsonify.js
+++ b/test/unit/helper/geojsonify.js
@@ -15,6 +15,8 @@ module.exports.tests.search = function(test, common) {
 
   var input = [
     {
+      "_id": "id1",
+      "_type": "type1",
       "center_point": {
         "lat": 51.5337144,
         "lon": -0.1069716
@@ -48,6 +50,8 @@ module.exports.tests.search = function(test, common) {
       }
     },
     {
+      "_id": "id2",
+      "_type": "type2",
       "type": "way",
       "name": {
         "default": "Blues Cafe"
@@ -90,6 +94,9 @@ module.exports.tests.search = function(test, common) {
           ]
         },
         "properties": {
+          "id": "id1",
+          "type": "type1",
+          "layer": "type1",
           "text": "'Round Midnight Jazz and Blues Bar, Angel, United Kingdom",
           "name": "'Round Midnight Jazz and Blues Bar",
           "alpha3": "GBR",
@@ -112,6 +119,9 @@ module.exports.tests.search = function(test, common) {
           ]
         },
         "properties": {
+          "id": "id2",
+          "type": "type2",
+          "layer": "type2",
           "text": "Blues Cafe, Smithfield, United Kingdom",
           "name": "Blues Cafe",
           "alpha3": "GBR",
diff --git a/test/unit/mock/backend.js b/test/unit/mock/backend.js
index f2759168..91a67b00 100644
--- a/test/unit/mock/backend.js
+++ b/test/unit/mock/backend.js
@@ -9,6 +9,8 @@ responses['client/suggest/fail/1'] = function( cmd, cb ){
 };
 responses['client/search/ok/1'] = function( cmd, cb ){
   return cb( undefined, searchEnvelope([{
+    _id: 'myid1',
+    _type: 'mytype1',
     _source: {
       value: 1,
       center_point: { lat: 100.1, lon: -50.5 },
@@ -16,6 +18,8 @@ responses['client/search/ok/1'] = function( cmd, cb ){
       admin0: 'country1', admin1: 'state1', admin2: 'city1'
     }
   }, {
+    _id: 'myid2',
+    _type: 'mytype2',
     _source: {
       value: 2,
       center_point: { lat: 100.2, lon: -51.5 },
@@ -26,6 +30,9 @@ responses['client/search/ok/1'] = function( cmd, cb ){
 };
 responses['client/mget/ok/1'] = function( cmd, cb ){
   return cb( undefined, mgetEnvelope([{
+    _id: 'myid1',
+    _type: 'mytype1',
+    found: true,
     _source: {
       value: 1,
       center_point: { lat: 100.1, lon: -50.5 },
@@ -33,6 +40,9 @@ responses['client/mget/ok/1'] = function( cmd, cb ){
       admin0: 'country1', admin1: 'state1', admin2: 'city1'
     }
   }, {
+    _id: 'myid2',
+    _type: 'mytype2',
+    found: true,
     _source: {
       value: 2,
       center_point: { lat: 100.2, lon: -51.5 },