Browse Source

Merge branch 'master' of github.com:pelias/api into missinglink

pull/475/head
missinglink 9 years ago
parent
commit
9a5d425cc8
  1. 54
      helper/geojsonify.js
  2. 2
      helper/labelSchema.js
  3. 11
      helper/placeTypes.js
  4. 56
      middleware/normalizeParentIds.js
  5. 40
      middleware/renamePlacenames.js
  6. 5
      package.json
  7. 7
      routes/v1.js
  8. 39
      test/unit/helper/geojsonify.js
  9. 15
      test/unit/helper/labelSchema.js
  10. 91
      test/unit/middleware/normalizeParentIds.js
  11. 1
      test/unit/run.js

54
helper/geojsonify.js

@ -4,6 +4,7 @@ var GeoJSON = require('geojson'),
labelGenerator = require('./labelGenerator'),
logger = require('pelias-logger').get('api'),
type_mapping = require('./type_mapping'),
Document = require('pelias-model').Document,
_ = require('lodash');
// Properties to be copied
@ -14,28 +15,31 @@ var DETAILS_PROPS = [
'confidence',
'distance',
'country',
'country_id',
'country_gid',
'country_a',
'macroregion',
'macroregion_id',
'macroregion_gid',
'macroregion_a',
'region',
'region_id',
'region_gid',
'region_a',
'macrocounty',
'macrocounty_id',
'macrocounty_gid',
'macrocounty_a',
'county',
'county_id',
'county_gid',
'county_a',
'localadmin',
'localadmin_id',
'localadmin_gid',
'localadmin_a',
'locality',
'locality_id',
'locality_gid',
'locality_a',
'borough',
'borough_gid',
'borough_a',
'neighbourhood',
'neighbourhood_id',
'neighbourhood_gid',
'bounding_box'
];
@ -65,6 +69,10 @@ function geojsonifyPlaces( docs ){
var geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] });
var geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] });
// to insert the bbox property at the top level of each feature, it must be done separately after
// initial geojson construction is finished
addBBoxPerFeature(geojson);
// bounding box calculations
computeBBox(geojson, geojsonExtentPoints);
@ -117,6 +125,30 @@ function addLabel(src, dst) {
dst.label = labelGenerator(dst);
}
/**
* Add bounding box
*
* @param {object} geojson
*/
function addBBoxPerFeature(geojson) {
geojson.features.forEach(function (feature) {
if (!feature.properties.hasOwnProperty('bounding_box')) {
return;
}
if (feature.properties.bounding_box) {
feature.bbox = [
feature.properties.bounding_box.min_lon,
feature.properties.bounding_box.min_lat,
feature.properties.bounding_box.max_lon,
feature.properties.bounding_box.max_lat
];
}
delete feature.properties.bounding_box;
});
}
/**
* Collect all points from the geodata.
@ -208,7 +240,8 @@ function copyProperties( source, props, dst ) {
* @param {object} src
*/
function makeGid(src) {
return lookupSource(src) + ':' + lookupLayer(src) + ':' + src._id;
var doc = new Document(lookupSource(src), lookupLayer(src), src._id);
return doc.getGid();
}
/**
@ -222,6 +255,9 @@ function addMetaData(src, dst) {
dst.gid = makeGid(src);
dst.layer = lookupLayer(src);
dst.source = lookupSource(src);
if (src.hasOwnProperty('bounding_box')) {
dst.bounding_box = src.bounding_box;
}
}
/**

2
helper/labelSchema.js

@ -3,7 +3,7 @@ var _ = require('lodash'),
module.exports = {
'USA': {
'local': getFirstProperty(['localadmin', 'locality', 'neighbourhood', 'county']),
'local': getFirstProperty(['borough', 'localadmin', 'locality', 'neighbourhood', 'county']),
'regional': getUsState,
'country': getFirstProperty(['country_a'])
},

11
helper/placeTypes.js

@ -0,0 +1,11 @@
module.exports = [
'country',
'macroregion',
'region',
'macrocounty',
'county',
'localadmin',
'locality',
'borough',
'neighbourhood'
];

56
middleware/normalizeParentIds.js

@ -0,0 +1,56 @@
var logger = require('pelias-logger').get('api');
var Document = require('pelias-model').Document;
var placeTypes = require('../helper/placeTypes');
/**
* Convert WOF integer ids to Pelias formatted ids that can be used by the /place endpoint.
* This should probably be moved to the import pipeline once we are happy with the way this works.
*/
function setup() {
return function (req, res, next) {
// do nothing if no result data set
if (!res || !res.data) {
return next();
}
res.data = res.data.map(normalizeParentIds);
next();
};
}
/**
* Update all parent ids in the admin hierarchy
*
* @param {object} place
* @return {object}
*/
function normalizeParentIds(place) {
if (place) {
placeTypes.forEach(function (placeType) {
if (place[placeType] && place[placeType].length > 0 && place[placeType][0]) {
place[placeType + '_gid'] = [ makeNewId(placeType, place[placeType + '_gid']) ];
}
});
}
return place;
}
/**
* Generate a valid Pelias ids from placetype and WOF id.
* Assumes all of the incoming ids are WOF ids.
*
* @param {string} placeType
* @param {number} id
* @return {string}
*/
function makeNewId(placeType, id) {
var doc = new Document('whosonfirst', placeType, id);
return doc.getGid();
}
module.exports = setup;

40
middleware/renamePlacenames.js

@ -1,16 +1,6 @@
var _ = require('lodash');
/**
- P is a preferred English name
- Q is a preferred name (in other languages)
- V is a well-known (but unofficial) variant for the place
(e.g. "New York City" for New York)
- S is either a synonym or a colloquial name for the place
(e.g. "Big Apple" for New York), or a version of the name which
is stripped of accent characters.
- A is an abbreviation or code for the place (e.g. "NYC" for New
York)
*/
var PARENT_PROPS = require('../helper/placeTypes');
var ADDRESS_PROPS = {
'number': 'housenumber',
@ -18,32 +8,6 @@ var ADDRESS_PROPS = {
'street': 'street'
};
var PARENT_PROPS = [
'country',
'country_id',
'country_a',
'macroregion',
'macroregion_id',
'macroregion_a',
'region',
'region_id',
'region_a',
'macrocounty',
'macrocounty_id',
'macrocounty_a',
'county',
'county_id',
'county_a',
'localadmin',
'localadmin_id',
'localadmin_a',
'locality',
'locality_id',
'locality_a',
'neighbourhood',
'neighbourhood_id'
];
function setup() {
return renamePlacenames;
@ -74,6 +38,8 @@ function renameOneRecord(place) {
if (place.parent) {
PARENT_PROPS.forEach(function (prop) {
place[prop] = place.parent[prop];
place[prop + '_a'] = place.parent[prop + '_a'];
place[prop + '_gid'] = place.parent[prop + '_id'];
});
}

5
package.json

@ -39,7 +39,7 @@
"async": "^1.5.2",
"check-types": "^6.0.0",
"cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve",
"elasticsearch": "^10.1.3",
"elasticsearch": "^11.0.0",
"express": "^4.8.8",
"express-http-proxy": "^0.6.0",
"extend": "3.0.0",
@ -53,6 +53,7 @@
"morgan": "1.7.0",
"pelias-config": "^1.0.1",
"pelias-logger": "^0.0.8",
"pelias-model": "^3.1.0",
"pelias-query": "6.2.0",
"pelias-suggester-pipeline": "2.0.4",
"stats-lite": "1.0.3",
@ -66,7 +67,7 @@
"nsp": "^2.2.0",
"precommit-hook": "^3.0.0",
"proxyquire": "^1.4.0",
"tap-dot": "1.0.4",
"tap-dot": "1.0.5",
"tape": "^4.4.0"
},
"pre-commit": [

7
routes/v1.js

@ -35,7 +35,8 @@ var postProc = {
renamePlacenames: require('../middleware/renamePlacenames'),
geocodeJSON: require('../middleware/geocodeJSON'),
sendJSON: require('../middleware/sendJSON'),
parseBoundingBox: require('../middleware/parseBBox')
parseBoundingBox: require('../middleware/parseBBox'),
normalizeParentIds: require('../middleware/normalizeParentIds')
};
/**
@ -67,6 +68,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -79,6 +81,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -94,6 +97,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),
@ -103,6 +107,7 @@ function addRoutes(app, peliasConfig) {
postProc.localNamingConventions(),
postProc.renamePlacenames(),
postProc.parseBoundingBox(),
postProc.normalizeParentIds(),
postProc.geocodeJSON(peliasConfig, base),
postProc.sendJSON
]),

39
test/unit/helper/geojsonify.js

@ -17,6 +17,8 @@ module.exports.tests.earth = function(test, common) {
var earth = [{
'_type': 'geoname',
'_id': '6295630',
'source': 'whosonfirst',
'layer': 'continent',
'name': {
'default': 'Earth'
},
@ -240,7 +242,7 @@ module.exports.tests.search = function(test, common) {
'country': [
'United States'
],
'country_id': [
'country_gid': [
'85633793'
],
'country_a': [
@ -249,7 +251,7 @@ module.exports.tests.search = function(test, common) {
'macroregion': [
'MacroRegion Name'
],
'macroregion_id': [
'macroregion_gid': [
'MacroRegion Id'
],
'macroregion_a': [
@ -258,7 +260,7 @@ module.exports.tests.search = function(test, common) {
'region': [
'New York'
],
'region_id': [
'region_gid': [
'85688543'
],
'region_a': [
@ -267,7 +269,7 @@ module.exports.tests.search = function(test, common) {
'macrocounty': [
'MacroCounty Name'
],
'macrocounty_id': [
'macrocounty_gid': [
'MacroCounty Id'
],
'macrocounty_a': [
@ -276,7 +278,7 @@ module.exports.tests.search = function(test, common) {
'county': [
'Kings County'
],
'county_id': [
'county_gid': [
'102082361'
],
'county_a': [
@ -285,20 +287,20 @@ module.exports.tests.search = function(test, common) {
'localadmin': [
'Brooklyn'
],
'localadmin_id': [
'localadmin_gid': [
'404521211'
],
'localadmin_a': [
null
],
'locality_id': [
'locality_gid': [
'85977539'
],
'locality_a': [
null
],
'neighbourhood': [],
'neighbourhood_id': []
'neighbourhood_gid': []
}
];
@ -316,31 +318,26 @@ module.exports.tests.search = function(test, common) {
'name': 'East New York',
'confidence': 0.888,
'country': 'United States',
'country_id': '85633793',
'country_gid': '85633793',
'country_a': 'USA',
'macroregion': 'MacroRegion Name',
'macroregion_id': 'MacroRegion Id',
'macroregion_gid': 'MacroRegion Id',
'macroregion_a': 'MacroRegion Abbreviation',
'region': 'New York',
'region_id': '85688543',
'region_gid': '85688543',
'region_a': 'NY',
'macrocounty': 'MacroCounty Name',
'macrocounty_id': 'MacroCounty Id',
'macrocounty_gid': 'MacroCounty Id',
'macrocounty_a': 'MacroCounty Abbreviation',
'county': 'Kings County',
'county_id': '102082361',
'county_gid': '102082361',
'localadmin': 'Brooklyn',
'localadmin_id': '404521211',
'localadmin_gid': '404521211',
'locality': 'New York',
'locality_id': '85977539',
'bounding_box': {
'min_lat': 40.6514712164,
'max_lat': 40.6737320588,
'min_lon': -73.8967895508,
'max_lon': -73.8665771484
},
'locality_gid': '85977539',
'label': 'East New York, Brooklyn, NY, USA'
},
'bbox': [-73.8967895508,40.6514712164,-73.8665771484,40.6737320588],
'geometry': {
'type': 'Point',
'coordinates': [

15
test/unit/helper/labelSchema.js

@ -772,6 +772,21 @@ module.exports.tests.default = function(test, common) {
});
test('USA.local borough should be shown if available', function(t) {
var record = {
borough: 'borough value',
region: 'region value',
region_a: 'region_a value',
country_a: 'country_a value'
};
var labelParts = ['initial value'];
var f = schemas.USA.local;
t.deepEqual(f(record, labelParts), ['initial value', 'borough value']);
t.end();
});
};
module.exports.all = function (tape, common) {

91
test/unit/middleware/normalizeParentIds.js

@ -0,0 +1,91 @@
var normalizer = require('../../../middleware/normalizeParentIds')();
module.exports.tests = {};
module.exports.tests.interface = function(test, common) {
test('WOF ids converted to Pelias ids', function(t) {
var input = {
data: [{
'parent': {
'country': ['United States'], // these shouldn't change
'country_id': ['85633793'],
'country_a': ['USA']
},
'country': ['United States'],
'country_gid': ['85633793'],
'country_a': ['USA'],
'macroregion': ['MacroRegion Name'],
'macroregion_gid': ['foobar'],
'macroregion_a': ['MacroRegion Abbreviation'],
'region': ['New York'],
'region_gid': ['85688543'],
'region_a': ['NY'],
'macrocounty': ['MacroCounty Name'],
'macrocounty_gid': ['~~~~~'],
'macrocounty_a': ['MacroCounty Abbreviation'],
'county': ['Kings County'],
'county_gid': ['102082361'],
'county_a': [null],
'localadmin': ['Brooklyn'],
'localadmin_gid': ['404521211'],
'localadmin_a': [null],
'locality': ['Some Locality'],
'locality_gid': ['85977539'],
'locality_a': [null],
'neighbourhood': [],
'neighbourhood_gid': []
}]
};
var expected = {
data: [{
'parent': {
'country': ['United States'],
'country_id': ['85633793'],
'country_a': ['USA']
},
'country': ['United States'],
'country_gid': ['whosonfirst:country:85633793'],
'country_a': ['USA'],
'macroregion': ['MacroRegion Name'],
'macroregion_gid': ['whosonfirst:macroregion:foobar'],
'macroregion_a': ['MacroRegion Abbreviation'],
'region': ['New York'],
'region_gid': ['whosonfirst:region:85688543'],
'region_a': ['NY'],
'macrocounty': ['MacroCounty Name'],
'macrocounty_gid': ['whosonfirst:macrocounty:~~~~~'],
'macrocounty_a': ['MacroCounty Abbreviation'],
'county': ['Kings County'],
'county_gid': ['whosonfirst:county:102082361'],
'county_a': [null],
'localadmin': ['Brooklyn'],
'localadmin_gid': ['whosonfirst:localadmin:404521211'],
'localadmin_a': [null],
'locality': ['Some Locality'],
'locality_gid': ['whosonfirst:locality:85977539'],
'locality_a': [null],
'neighbourhood': [],
'neighbourhood_gid': []
}]
};
normalizer({}, input, function () {
t.deepEqual(input, expected);
t.end();
});
});
};
module.exports.all = function (tape, common) {
function test(name, testFunction) {
return tape('[middleware] normalizeParentIds: ' + name, testFunction);
}
for( var testCase in module.exports.tests ){
module.exports.tests[testCase](test, common);
}
};

1
test/unit/run.js

@ -28,6 +28,7 @@ var tests = [
require('./middleware/localNamingConventions'),
require('./middleware/dedupe'),
require('./middleware/parseBBox'),
require('./middleware/normalizeParentIds'),
require('./query/autocomplete'),
require('./query/autocomplete_defaults'),
require('./query/search_defaults'),

Loading…
Cancel
Save