Browse Source

Merge pull request #488 from pelias/normalize-parent-ids

Convert parent WOF ids to valid Pelias ids to be used by /place
pull/490/head
Diana Shkolnikov 9 years ago
parent
commit
277153d6ea
  1. 21
      helper/geojsonify.js
  2. 11
      helper/placeTypes.js
  3. 56
      middleware/normalizeParentIds.js
  4. 40
      middleware/renamePlacenames.js
  5. 1
      package.json
  6. 7
      routes/v1.js
  7. 32
      test/unit/helper/geojsonify.js
  8. 91
      test/unit/middleware/normalizeParentIds.js
  9. 1
      test/unit/run.js

21
helper/geojsonify.js

@ -4,6 +4,7 @@ var GeoJSON = require('geojson'),
labelGenerator = require('./labelGenerator'),
logger = require('pelias-logger').get('api'),
type_mapping = require('./type_mapping'),
Document = require('pelias-model').Document,
_ = require('lodash');
// Properties to be copied
@ -14,28 +15,29 @@ var DETAILS_PROPS = [
'confidence',
'distance',
'country',
'country_id',
'country_gid',
'country_a',
'macroregion',
'macroregion_id',
'macroregion_gid',
'macroregion_a',
'region',
'region_id',
'region_gid',
'region_a',
'macrocounty',
'macrocounty_id',
'macrocounty_gid',
'macrocounty_a',
'county',
'county_id',
'county_gid',
'county_a',
'localadmin',
'localadmin_id',
'localadmin_gid',
'localadmin_a',
'locality',
'locality_id',
'locality_gid',
'locality_a',
'neighbourhood',
'neighbourhood_id'
'neighbourhood_gid',
'bounding_box'
];
@ -235,7 +237,8 @@ function copyProperties( source, props, dst ) {
* @param {object} src
*/
function makeGid(src) {
return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id;
var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
return doc.getGid();
}
/**

11
helper/placeTypes.js

@ -0,0 +1,11 @@
module.exports = [
'country',
'macroregion',
'region',
'macrocounty',
'county',
'localadmin',
'locality',
'borough',
'neighbourhood'
];

56
middleware/normalizeParentIds.js

@ -0,0 +1,56 @@
var logger = require('pelias-logger').get('api');
var Document = require('pelias-model').Document;
var placeTypes = require('../helper/placeTypes');
/**
* Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint.
* This should probably be moved to the import pipeline once we are happy with the way this works.
*/
function setup() {
return function (req, res, next) {
// do nothing if no result data set
if (!res || !res.data) {
return next();
}
res.data = res.data.map(normalizeParentIds);
next();
};
}
/**
* Update all parent ids in the admin hierarchy
*
* @param {object} place
* @return {object}
*/
function normalizeParentIds(place) {
if (place) {
placeTypes.forEach(function (placeType) {
if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) {
place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ];
}
});
}
return place;
}
/**
* Generate a valid Pelias ids from placetype and WOF id.
* Assumes all of the incoming ids are WOF ids.
*
* @param {string} placeType
* @param {number} id
* @return {string}
*/
function makeNewId(placeType, id) {
var doc = new Document('whosonfirst', placeType, id);
return doc.getGid();
}
module.exports = setup;

40
middleware/renamePlacenames.js

@ -1,16 +1,6 @@
var _ = require('lodash');
/**
- P is a preferred English name
- Q is a preferred name (in other languages)
- V is a well-known (but unofficial) variant for the place
(e.g. "New York City" for New York)
- S is either a synonym or a colloquial name for the place
(e.g. "Big Apple" for New York), or a version of the name which
is stripped of accent characters.
- A is an abbreviation or code for the place (e.g. "NYC" for New
York)
*/
var PARENT_PROPS = require('../helper/placeTypes');
var ADDRESS_PROPS = {
'number': 'housenumber',
@ -18,32 +8,6 @@ var ADDRESS_PROPS = {
'street': 'street'
};
var PARENT_PROPS = [
'country',
'country_id',
'country_a',
'macroregion',
'macroregion_id',
'macroregion_a',
'region',
'region_id',
'region_a',
'macrocounty',
'macrocounty_id',
'macrocounty_a',
'county',
'county_id',
'county_a',
'localadmin',
'localadmin_id',
'localadmin_a',
'locality',
'locality_id',
'locality_a',
'neighbourhood',
'neighbourhood_id'
];
function setup() {
return renamePlacenames;
@ -74,6 +38,8 @@ function renameOneRecord(place) {
if (place.parent) {
PARENT_PROPS.forEach(function (prop) {
place[prop] = place.parent[prop];
place[prop + '_a'] = place.parent[prop + '_a'];
place[prop + '_gid'] = place.parent[prop + '_id'];
});
}

1
package.json

@ -53,6 +53,7 @@
"morgan": "1.7.0",
"pelias-config": "^1.0.1",
"pelias-logger": "^0.0.8",
"pelias-model": "^3.1.0",
"pelias-query": "6.2.0",
"pelias-suggester-pipeline": "2.0.4",
"stats-lite": "1.0.3",

7
routes/v1.js

@ -35,7 +35,8 @@ var postProc = {
renamePlacenames: require('../middleware/renamePlacenames'),
geocodeJSON: require('../middleware/geocodeJSON'),
sendJSON: require('../middleware/sendJSON'),
parseBoundingBox: require('../middleware/parseBBox')
parseBoundingBox: require('../middleware/parseBBox'),
normalizeParentIds: require('../middleware/normalizeParentIds')
};
/**
@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -79,6 +81,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -94,6 +97,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -103,6 +107,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),

32
test/unit/helper/geojsonify.js

@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) {
var earth = [{
'_type': 'geoname',
'_id': '6295630',
'source': 'whosonfirst',
'layer': 'continent',
'name': {
'default': 'Earth'
},
@ -240,7 +242,7 @@ module.exports.tests.search = function(test, common) {
'country': [
'United States'
],
'country_id': [
'country_gid': [
'85633793'
],
'country_a': [
@ -249,7 +251,7 @@ module.exports.tests.search = function(test, common) {
'macroregion': [
'MacroRegion Name'
],
'macroregion_id': [
'macroregion_gid': [
'MacroRegion Id'
],
'macroregion_a': [
@ -258,7 +260,7 @@ module.exports.tests.search = function(test, common) {
'region': [
'New York'
],
'region_id': [
'region_gid': [
'85688543'
],
'region_a': [
@ -267,7 +269,7 @@ module.exports.tests.search = function(test, common) {
'macrocounty': [
'MacroCounty Name'
],
'macrocounty_id': [
'macrocounty_gid': [
'MacroCounty Id'
],
'macrocounty_a': [
@ -276,7 +278,7 @@ module.exports.tests.search = function(test, common) {
'county': [
'Kings County'
],
'county_id': [
'county_gid': [
'102082361'
],
'county_a': [
@ -285,20 +287,20 @@ module.exports.tests.search = function(test, common) {
'localadmin': [
'Brooklyn'
],
'localadmin_id': [
'localadmin_gid': [
'404521211'
],
'localadmin_a': [
null
],
'locality_id': [
'locality_gid': [
'85977539'
],
'locality_a': [
null
],
'neighbourhood': [],
'neighbourhood_id': []
'neighbourhood_gid': []
}
];
@ -316,23 +318,23 @@ module.exports.tests.search = function(test, common) {
'name': 'East New York',
'confidence': 0.888,
'country': 'United States',
'country_id': '85633793',
'country_gid': '85633793',
'country_a': 'USA',
'macroregion': 'MacroRegion Name',
'macroregion_id': 'MacroRegion Id',
'macroregion_gid': 'MacroRegion Id',
'macroregion_a': 'MacroRegion Abbreviation',
'region': 'New York',
'region_id': '85688543',
'region_gid': '85688543',
'region_a': 'NY',
'macrocounty': 'MacroCounty Name',
'macrocounty_id': 'MacroCounty Id',
'macrocounty_gid': 'MacroCounty Id',
'macrocounty_a': 'MacroCounty Abbreviation',
'county': 'Kings County',
'county_id': '102082361',
'county_gid': '102082361',
'localadmin': 'Brooklyn',
'localadmin_id': '404521211',
'localadmin_gid': '404521211',
'locality': 'New York',
'locality_id': '85977539',
'locality_gid': '85977539',
'label': 'East New York, Brooklyn, NY, USA'
},
'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],

91
test/unit/middleware/normalizeParentIds.js

@ -0,0 +1,91 @@
var normalizer = require('../../../middleware/normalizeParentIds')();
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('WOF ids converted to Pelias ids', function(t) {
var input = {
data: [{
'parent': {
'country': ['United States'], // these shouldn't change
'country_id': ['85633793'],
'country_a': ['USA']
},
'country': ['United States'],
'country_gid': ['85633793'],
'country_a': ['USA'],
'macroregion': ['MacroRegion Name'],
'macroregion_gid': ['foobar'],
'macroregion_a': ['MacroRegion Abbreviation'],
'region': ['New York'],
'region_gid': ['85688543'],
'region_a': ['NY'],
'macrocounty': ['MacroCounty Name'],
'macrocounty_gid': ['~~~~~'],
'macrocounty_a': ['MacroCounty Abbreviation'],
'county': ['Kings County'],
'county_gid': ['102082361'],
'county_a': [null],
'localadmin': ['Brooklyn'],
'localadmin_gid': ['404521211'],
'localadmin_a': [null],
'locality': ['Some Locality'],
'locality_gid': ['85977539'],
'locality_a': [null],
'neighbourhood': [],
'neighbourhood_gid': []
}]
};
var expected = {
data: [{
'parent': {
'country': ['United States'],
'country_id': ['85633793'],
'country_a': ['USA']
},
'country': ['United States'],
'country_gid': ['whosonfirst:country:85633793'],
'country_a': ['USA'],
'macroregion': ['MacroRegion Name'],
'macroregion_gid': ['whosonfirst:macroregion:foobar'],
'macroregion_a': ['MacroRegion Abbreviation'],
'region': ['New York'],
'region_gid': ['whosonfirst:region:85688543'],
'region_a': ['NY'],
'macrocounty': ['MacroCounty Name'],
'macrocounty_gid': ['whosonfirst:macrocounty:~~~~~'],
'macrocounty_a': ['MacroCounty Abbreviation'],
'county': ['Kings County'],
'county_gid': ['whosonfirst:county:102082361'],
'county_a': [null],
'localadmin': ['Brooklyn'],
'localadmin_gid': ['whosonfirst:localadmin:404521211'],
'localadmin_a': [null],
'locality': ['Some Locality'],
'locality_gid': ['whosonfirst:locality:85977539'],
'locality_a': [null],
'neighbourhood': [],
'neighbourhood_gid': []
}]
};
normalizer({}, input, function () {
t.deepEqual(input, expected);
t.end();
});
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('[middleware] normalizeParentIds: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

1
test/unit/run.js

@ -28,6 +28,7 @@ var tests = [
require('./middleware/localNamingConventions'),
require('./middleware/dedupe'),
require('./middleware/parseBBox'),
require('./middleware/normalizeParentIds'),
require('./query/autocomplete'),
require('./query/autocomplete_defaults'),
require('./query/search_defaults'),

Loading…
Cancel
Save