Browse Source

Merge pull request #444 from vesameskanen/fix-dedupe

Fix dedupe
pull/462/head
Diana Shkolnikov 9 years ago
parent
commit
5919901b7a
  1. 14
      middleware/dedupe.js
  2. 54
      test/unit/fixture/dedupe_elasticsearch_results.js
  3. 2
      test/unit/middleware/dedupe.js

14
middleware/dedupe.js

@ -42,13 +42,17 @@ function isDifferent(item1, item2) {
if (item1.hasOwnProperty('parent') && item2.hasOwnProperty('parent')) {
propMatch(item1.parent, item2.parent, 'region_a');
propMatch(item1.parent, item2.parent, 'country');
propMatch(item1.parent, item2.parent, 'locality');
propMatch(item1.parent, item2.parent, 'neighbourhood');
}
else if (item1.parent !== item2.parent) {
throw new Error('different');
}
if (item1.hasOwnProperty('name') && item2.hasOwnProperty('name')) {
propMatch(item1.name, item2.name, 'default');
for (var lang in item1.name) {
propMatch(item1.name, item2.name, lang);
}
}
else {
propMatch(item1, item2, 'name');
@ -87,10 +91,8 @@ function propMatch(item1, item2, prop) {
// in the case the property is an array (currently only in parent schema)
// simply take the 1st item. this will change in the near future to support multiple hierarchies
if (_.isArray(prop1) && _.isArray(prop2)) {
prop1 = prop1[0];
prop2= prop2[0];
}
if (_.isArray(prop1)) { prop1 = prop1[0]; }
if (_.isArray(prop2)) { prop2 = prop2[0]; }
if (normalizeString(prop1) !== normalizeString(prop2)) {
throw new Error('different');
@ -111,4 +113,4 @@ function normalizeString(str) {
}
module.exports = setup;
module.exports = setup;

54
test/unit/fixture/dedupe_elasticsearch_results.js

@ -26,6 +26,60 @@ module.exports = [
'_score': 1.2367082,
'confidence': 0.879
},
{ // same as above, but change the neighbourhood
'center_point': {
'lon': -77.207456,
'lat': 41.039265
},
'address': {},
'parent': {
'localadmin': 'East Lampeter',
'region_a': 'PA',
'region': 'Pennsylvania',
'locality': 'Smoketown',
'country_a': 'USA',
'county': 'Lancaster County',
'country': 'United States',
'neighbourhood': 'Blueland' // ###
},
'name': {
'default': 'East Lampeter High School'
},
'category': [
'education'
],
'_id': '357321757',
'_type': 'venue',
'_score': 1.2367082,
'confidence': 0.879
},
{ // same as #1, but change the locality
'center_point': {
'lon': -73.207456,
'lat': 42.039265
},
'address': {},
'parent': {
'localadmin': 'East Lampeter',
'region_a': 'PA',
'region': 'Pennsylvania',
'locality': 'Firetown', // ###
'country_a': 'USA',
'county': 'Lancaster County',
'country': 'United States',
'neighbourhood': 'Greenland'
},
'name': {
'default': 'East Lampeter High School'
},
'category': [
'education'
],
'_id': '357321757',
'_type': 'venue',
'_score': 1.2367082,
'confidence': 0.879
},
{
'center_point': {
'lon': -76.207456,

2
test/unit/middleware/dedupe.js

@ -16,7 +16,7 @@ module.exports.tests.dedupe = function(test, common) {
data: data
};
var expectedCount = 7;
var expectedCount = 9;
dedupe(req, res, function () {
t.equal(res.data.length, expectedCount, 'results have fewer items than before');
t.end();

Loading…
Cancel
Save