api/middleware/dedupe.js

const logger = require('pelias-logger').get('api');
const _ = require('lodash');
const isDifferent = require('../helper/diffPlaces').isDifferent;
const field = require('../helper/fieldValue');

function setup() {
  return dedupeResults;
}

function dedupeResults(req, res, next) {
  // do nothing if no result data set
  if (_.isUndefined(req.clean) || _.isUndefined(res) || _.isUndefined(res.data)) {
    return next();
  }

  // loop through data items and only copy unique items to uniqueResults
  var uniqueResults = [];

  _.some(res.data, function (hit) {

    if (_.isEmpty(uniqueResults)) {
      uniqueResults.push(hit);
    }
    else {
      // if there are multiple items in results, loop through them to find a dupe
      // save off the index of the dupe if found
      var dupeIndex = uniqueResults.findIndex(function (elem, index, array) {
        return !isDifferent(elem, hit);
      });

      // if a dupe is not found, just add to results and move on
      if (dupeIndex === -1) {
        uniqueResults.push(hit);
      }
      // if dupe was found, we need to check which of the records is preferred
      // since the order in which Elasticsearch returns identical text matches is arbitrary
      // of course, if the new one is preferred we should replace previous with new
      else if (isPreferred(uniqueResults[dupeIndex], hit)) {
        logger.info('[dupe][replacing]', {
          query: req.clean.text,
          previous: uniqueResults[dupeIndex].source,
          hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id
        });
        // replace previous dupe item with current hit
        uniqueResults[dupeIndex] = hit;
      }
      // if not preferred over existing, just log and move on
      else {
        logger.info('[dupe][skipping]', {
          query: req.clean.text,
          previous: uniqueResults[dupeIndex].source,
          hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id
        });
      }
    }

    // stop looping when requested size has been reached in uniqueResults
    return req.clean.size <= uniqueResults.length;
  });

  res.data = uniqueResults;

  next();
}

function isPreferred(existing, candidateReplacement) {
  // NOTE: we are assuming here that the layer for both records is the same
  const hasZip = _.bind(_.has, null, _.bind.placeholder, 'address_parts.zip');

  // https://github.com/pelias/api/issues/872
  const candidateHasZip = hasZip(candidateReplacement);
  const existingHasZip = hasZip(existing);
  if (candidateHasZip !== existingHasZip) {
    return candidateHasZip;
  }

  //bind the trumps function to the data items to keep the rest of the function clean
  var trumpsFunc = trumps.bind(null, existing, candidateReplacement);

  return trumpsFunc('geonames', 'whosonfirst') || // WOF has bbox and is generally preferred
         trumpsFunc('openstreetmap', 'openaddresses') || // addresses are better in OA
         trumpsFunc('whosonfirst', 'openstreetmap'); // venues are better in OSM, at this time
}

function trumps(existing, candidateReplacement, loserSource, winnerSource) {
  return existing.source === loserSource && candidateReplacement.source === winnerSource;
}

module.exports = setup;