Merge pull request #488 from pelias/normalize-parent-ids

Convert parent WOF ids to valid Pelias ids to be used by /place

helper/geojsonify.js

@@ -4,6 +4,7 @@ var GeoJSON = require('geojson'),
     labelGenerator = require('./labelGenerator'),
     logger = require('pelias-logger').get('api'),
     type_mapping = require('./type_mapping'),
+    Document = require('pelias-model').Document,
     _ = require('lodash');
 
 // Properties to be copied
@@ -14,28 +15,29 @@ var DETAILS_PROPS = [
   'confidence',
   'distance',
   'country',
-  'country_id',
+  'country_gid',
   'country_a',
   'macroregion',
-  'macroregion_id',
+  'macroregion_gid',
   'macroregion_a',
   'region',
-  'region_id',
+  'region_gid',
   'region_a',
   'macrocounty',
-  'macrocounty_id',
+  'macrocounty_gid',
   'macrocounty_a',
   'county',
-  'county_id',
+  'county_gid',
   'county_a',
   'localadmin',
-  'localadmin_id',
+  'localadmin_gid',
   'localadmin_a',
   'locality',
-  'locality_id',
+  'locality_gid',
   'locality_a',
   'neighbourhood',
-  'neighbourhood_id'
+  'neighbourhood_gid',
+  'bounding_box'
 ];
 
 
@@ -235,7 +237,8 @@ function copyProperties( source, props, dst ) {
  * @param {object} src
  */
 function makeGid(src) {
-  return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id;
+  var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
+  return doc.getGid();
 }
 
 /**

helper/placeTypes.js

@@ -0,0 +1,11 @@
+module.exports = [
+  'country',
+  'macroregion',
+  'region',
+  'macrocounty',
+  'county',
+  'localadmin',
+  'locality',
+  'borough',
+  'neighbourhood'
+];

middleware/normalizeParentIds.js

@@ -0,0 +1,56 @@
+var logger = require('pelias-logger').get('api');
+var Document = require('pelias-model').Document;
+
+var placeTypes = require('../helper/placeTypes');
+
+/**
+ * Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint.
+ * This should probably be moved to the import pipeline once we are happy with the way this works.
+ */
+
+function setup() {
+  return function (req, res, next) {
+    // do nothing if no result data set
+    if (!res || !res.data) {
+      return next();
+    }
+
+    res.data = res.data.map(normalizeParentIds);
+
+    next();
+  };
+}
+
+/**
+ * Update all parent ids in the admin hierarchy
+ *
+ * @param {object} place
+ * @return {object}
+ */
+function normalizeParentIds(place) {
+
+  if (place) {
+    placeTypes.forEach(function (placeType) {
+      if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) {
+        place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ];
+      }
+    });
+  }
+
+  return place;
+}
+
+/**
+ * Generate a valid Pelias ids from placetype and WOF id.
+ * Assumes all of the incoming ids are WOF ids.
+ *
+ * @param {string} placeType
+ * @param {number} id
+ * @return {string}
+ */
+function makeNewId(placeType, id) {
+  var doc = new Document('whosonfirst', placeType, id);
+  return doc.getGid();
+}
+
+module.exports = setup;

middleware/renamePlacenames.js

@@ -1,16 +1,6 @@
 var _ = require('lodash');
 
-/**
+var PARENT_PROPS = require('../helper/placeTypes');
- - P is a preferred English name
- - Q is a preferred name (in other languages)
- - V is a well-known (but unofficial) variant for the place
- (e.g. "New York City" for New York)
- - S is either a synonym or a colloquial name for the place
- (e.g. "Big Apple" for New York), or a version of the name which
- is stripped of accent characters.
- - A is an abbreviation or code for the place (e.g. "NYC" for New
- York)
- */
 
 var ADDRESS_PROPS = {
   'number': 'housenumber',
@@ -18,32 +8,6 @@ var ADDRESS_PROPS = {
   'street': 'street'
 };
 
-var PARENT_PROPS = [
-  'country',
-  'country_id',
-  'country_a',
-  'macroregion',
-  'macroregion_id',
-  'macroregion_a',
-  'region',
-  'region_id',
-  'region_a',
-  'macrocounty',
-  'macrocounty_id',
-  'macrocounty_a',
-  'county',
-  'county_id',
-  'county_a',
-  'localadmin',
-  'localadmin_id',
-  'localadmin_a',
-  'locality',
-  'locality_id',
-  'locality_a',
-  'neighbourhood',
-  'neighbourhood_id'
-];
-
 
 function setup() {
   return renamePlacenames;
@@ -74,6 +38,8 @@ function renameOneRecord(place) {
   if (place.parent) {
     PARENT_PROPS.forEach(function (prop) {
       place[prop] = place.parent[prop];
+      place[prop + '_a'] = place.parent[prop + '_a'];
+      place[prop + '_gid'] = place.parent[prop + '_id'];
     });
   }
 

package.json

@@ -53,6 +53,7 @@
     "morgan": "1.7.0",
     "pelias-config": "^1.0.1",
     "pelias-logger": "^0.0.8",
+    "pelias-model": "^3.1.0",
     "pelias-query": "6.2.0",
     "pelias-suggester-pipeline": "2.0.4",
     "stats-lite": "1.0.3",

routes/v1.js

@@ -35,7 +35,8 @@ var postProc = {
   renamePlacenames: require('../middleware/renamePlacenames'),
   geocodeJSON: require('../middleware/geocodeJSON'),
   sendJSON: require('../middleware/sendJSON'),
-  parseBoundingBox: require('../middleware/parseBBox')
+  parseBoundingBox: require('../middleware/parseBBox'),
+  normalizeParentIds: require('../middleware/normalizeParentIds')
 };
 
 /**
@@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) {
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
@@ -79,6 +81,7 @@ function addRoutes(app, peliasConfig) {
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
@@ -94,6 +97,7 @@ function addRoutes(app, peliasConfig) {
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),
@@ -103,6 +107,7 @@ function addRoutes(app, peliasConfig) {
       postProc.localNamingConventions(),
       postProc.renamePlacenames(),
       postProc.parseBoundingBox(),
+      postProc.normalizeParentIds(),
       postProc.geocodeJSON(peliasConfig, base),
       postProc.sendJSON
     ]),

test/unit/helper/geojsonify.js

@@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) {
   var earth = [{
     '_type': 'geoname',
     '_id': '6295630',
+    'source': 'whosonfirst',
+    'layer': 'continent',
     'name': {
       'default': 'Earth'
     },
@@ -240,7 +242,7 @@ module.exports.tests.search = function(test, common) {
         'country': [
           'United States'
         ],
-        'country_id': [
+        'country_gid': [
           '85633793'
         ],
         'country_a': [
@@ -249,7 +251,7 @@ module.exports.tests.search = function(test, common) {
         'macroregion': [
           'MacroRegion Name'
         ],
-        'macroregion_id': [
+        'macroregion_gid': [
           'MacroRegion Id'
         ],
         'macroregion_a': [
@@ -258,7 +260,7 @@ module.exports.tests.search = function(test, common) {
         'region': [
           'New York'
         ],
-        'region_id': [
+        'region_gid': [
           '85688543'
         ],
         'region_a': [
@@ -267,7 +269,7 @@ module.exports.tests.search = function(test, common) {
         'macrocounty': [
           'MacroCounty Name'
         ],
-        'macrocounty_id': [
+        'macrocounty_gid': [
           'MacroCounty Id'
         ],
         'macrocounty_a': [
@@ -276,7 +278,7 @@ module.exports.tests.search = function(test, common) {
         'county': [
           'Kings County'
         ],
-        'county_id': [
+        'county_gid': [
           '102082361'
         ],
         'county_a': [
@@ -285,20 +287,20 @@ module.exports.tests.search = function(test, common) {
         'localadmin': [
           'Brooklyn'
         ],
-        'localadmin_id': [
+        'localadmin_gid': [
           '404521211'
         ],
         'localadmin_a': [
           null
         ],
-        'locality_id': [
+        'locality_gid': [
           '85977539'
         ],
         'locality_a': [
           null
         ],
         'neighbourhood': [],
-        'neighbourhood_id': []
+        'neighbourhood_gid': []
       }
     ];
 
@@ -316,23 +318,23 @@ module.exports.tests.search = function(test, common) {
             'name': 'East New York',
             'confidence': 0.888,
             'country': 'United States',
-            'country_id': '85633793',
+            'country_gid': '85633793',
             'country_a': 'USA',
             'macroregion': 'MacroRegion Name',
-            'macroregion_id': 'MacroRegion Id',
+            'macroregion_gid': 'MacroRegion Id',
             'macroregion_a': 'MacroRegion Abbreviation',
             'region': 'New York',
-            'region_id': '85688543',
+            'region_gid': '85688543',
             'region_a': 'NY',
             'macrocounty': 'MacroCounty Name',
-            'macrocounty_id': 'MacroCounty Id',
+            'macrocounty_gid': 'MacroCounty Id',
             'macrocounty_a': 'MacroCounty Abbreviation',
             'county': 'Kings County',
-            'county_id': '102082361',
+            'county_gid': '102082361',
             'localadmin': 'Brooklyn',
-            'localadmin_id': '404521211',
+            'localadmin_gid': '404521211',
             'locality': 'New York',
-            'locality_id': '85977539',
+            'locality_gid': '85977539',
             'label': 'East New York, Brooklyn, NY, USA'
           },
           'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],

test/unit/middleware/normalizeParentIds.js

@@ -0,0 +1,91 @@
+var normalizer = require('../../../middleware/normalizeParentIds')();
+
+module.exports.tests = {};
+
+module.exports.tests.interface = function(test, common) {
+  test('WOF ids converted to Pelias ids', function(t) {
+
+    var input = {
+      data: [{
+        'parent': {
+          'country': ['United States'], // these shouldn't change
+          'country_id': ['85633793'],
+          'country_a': ['USA']
+        },
+        'country': ['United States'],
+        'country_gid': ['85633793'],
+        'country_a': ['USA'],
+        'macroregion': ['MacroRegion Name'],
+        'macroregion_gid': ['foobar'],
+        'macroregion_a': ['MacroRegion Abbreviation'],
+        'region': ['New York'],
+        'region_gid': ['85688543'],
+        'region_a': ['NY'],
+        'macrocounty': ['MacroCounty Name'],
+        'macrocounty_gid': ['~~~~~'],
+        'macrocounty_a': ['MacroCounty Abbreviation'],
+        'county': ['Kings County'],
+        'county_gid': ['102082361'],
+        'county_a': [null],
+        'localadmin': ['Brooklyn'],
+        'localadmin_gid': ['404521211'],
+        'localadmin_a': [null],
+        'locality': ['Some Locality'],
+        'locality_gid': ['85977539'],
+        'locality_a': [null],
+        'neighbourhood': [],
+        'neighbourhood_gid': []
+      }]
+    };
+
+    var expected = {
+      data: [{
+        'parent': {
+          'country': ['United States'],
+          'country_id': ['85633793'],
+          'country_a': ['USA']
+        },
+        'country': ['United States'],
+        'country_gid': ['whosonfirst:country:85633793'],
+        'country_a': ['USA'],
+        'macroregion': ['MacroRegion Name'],
+        'macroregion_gid': ['whosonfirst:macroregion:foobar'],
+        'macroregion_a': ['MacroRegion Abbreviation'],
+        'region': ['New York'],
+        'region_gid': ['whosonfirst:region:85688543'],
+        'region_a': ['NY'],
+        'macrocounty': ['MacroCounty Name'],
+        'macrocounty_gid': ['whosonfirst:macrocounty:~~~~~'],
+        'macrocounty_a': ['MacroCounty Abbreviation'],
+        'county': ['Kings County'],
+        'county_gid': ['whosonfirst:county:102082361'],
+        'county_a': [null],
+        'localadmin': ['Brooklyn'],
+        'localadmin_gid': ['whosonfirst:localadmin:404521211'],
+        'localadmin_a': [null],
+        'locality': ['Some Locality'],
+        'locality_gid': ['whosonfirst:locality:85977539'],
+        'locality_a': [null],
+        'neighbourhood': [],
+        'neighbourhood_gid': []
+      }]
+    };
+
+    normalizer({}, input, function () {
+      t.deepEqual(input, expected);
+      t.end();
+    });
+
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('[middleware] normalizeParentIds: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/unit/run.js

@@ -28,6 +28,7 @@ var tests = [
   require('./middleware/localNamingConventions'),
   require('./middleware/dedupe'),
   require('./middleware/parseBBox'),
+  require('./middleware/normalizeParentIds'),
   require('./query/autocomplete'),
   require('./query/autocomplete_defaults'),
   require('./query/search_defaults'),