From 2e92f15a918adbd3904cb33e4c4715df5a544f54 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:51:44 -0400 Subject: [PATCH 1/8] Whitespace --- middleware/sizeCalculator.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index 69409e45..f8c6936c 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -8,15 +8,15 @@ var MIN_QUERY_SIZE = 20; * Utility for calculating query result size * incorporating padding for dedupe process */ -function setup() { - return function setQuerySize(req, res, next) { - if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { - return next(); - } +function setup(min_size) { + return function setQuerySize(req, res, next) { + if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { + return next(); + } - req.clean.querySize = calculateSize(req.clean.size); - next(); - }; + req.clean.querySize = calculateSize(req.clean.size, min_size); + next(); + }; } /** From c966e6f6aed5c5ce6126413258024782301a434c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:52:01 -0400 Subject: [PATCH 2/8] Allow the minimum Elasticearch query size to be set per endpoint Reverse queries don't need as much deduping, for example --- middleware/sizeCalculator.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index f8c6936c..9b577916 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -9,6 +9,10 @@ var MIN_QUERY_SIZE = 20; * incorporating padding for dedupe process */ function setup(min_size) { + if (min_size === undefined) { + min_size = MIN_QUERY_SIZE; + } + return function setQuerySize(req, res, next) { if (_.isUndefined(req.clean) || _.isUndefined(req.clean.size)) { return next(); @@ -25,8 +29,8 @@ function setup(min_size) { * @param {number} cleanSize * @returns {number} */ -function calculateSize(cleanSize) { - return Math.max(MIN_QUERY_SIZE, cleanSize * SIZE_PADDING); +function calculateSize(cleanSize, min_size) { + return Math.max(min_size, cleanSize * SIZE_PADDING); } module.exports = setup; From 07dd704e1d946ebc2ceb020878dd5b6157806542 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 11 Mar 2018 22:52:27 -0400 Subject: [PATCH 3/8] Set ES query minimum size to 2 for /v1/reverse --- routes/v1.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/v1.js b/routes/v1.js index f4e8176f..6b40f1d6 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -344,7 +344,7 @@ function addRoutes(app, peliasConfig) { reverse: createRouter([ sanitizers.reverse.middleware, middleware.requestLanguage, - middleware.calcSize(), + middleware.calcSize(2), controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute), controllers.coarse_reverse(pipService, coarseReverseShouldExecute), postProc.distances('point.'), From 5247ea575daf5dcfd5be222583892b89c3db595c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:15:27 -0400 Subject: [PATCH 4/8] Give variable a better name --- middleware/sizeCalculator.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/sizeCalculator.js b/middleware/sizeCalculator.js index 9b577916..f7ea906a 100644 --- a/middleware/sizeCalculator.js +++ b/middleware/sizeCalculator.js @@ -2,7 +2,7 @@ var _ = require('lodash'); var SIZE_PADDING = 2; -var MIN_QUERY_SIZE = 20; +var DEFAULT_MIN_QUERY_SIZE = 20; /** * Utility for calculating query result size @@ -10,7 +10,7 @@ var MIN_QUERY_SIZE = 20; */ function setup(min_size) { if (min_size === undefined) { - min_size = MIN_QUERY_SIZE; + min_size = DEFAULT_MIN_QUERY_SIZE; } return function setQuerySize(req, res, next) { From 825b506a7522473ba422af0711d60ab2b01a7785 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:22:08 -0400 Subject: [PATCH 5/8] Revise tests to account for sizeCalculator setup params --- test/unit/helper/sizeCalculator.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index 6c8d8d22..cbcd91b8 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -1,11 +1,10 @@ - -var calcSize = require('../../../middleware/sizeCalculator.js')(); +var calcSizeMiddleware = require('../../../middleware/sizeCalculator.js'); module.exports.tests = {}; module.exports.tests.interface = function(test, common) { test('interface', function(t) { - t.equal(typeof calcSize, 'function', 'valid function'); + t.equal(typeof calcSizeMiddleware, 'function', 'valid function'); t.end(); }); }; @@ -24,6 +23,7 @@ module.exports.tests.valid = function(test, common) { test('size=0', function (t) { setup(0); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -32,6 +32,7 @@ module.exports.tests.valid = function(test, common) { test('size=1', function (t) { setup(1); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -40,6 +41,7 @@ module.exports.tests.valid = function(test, common) { test('size=10', function (t) { setup(10); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); t.end(); @@ -48,6 +50,7 @@ module.exports.tests.valid = function(test, common) { test('size=20', function (t) { setup(20); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 40); t.end(); @@ -56,6 +59,7 @@ module.exports.tests.valid = function(test, common) { test('no size', function (t) { setup(); + const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); t.end(); From 94946989bb4c872085de452f10a5ed3a98409113 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:24:59 -0400 Subject: [PATCH 6/8] Test cases with a lower min query size --- test/unit/helper/sizeCalculator.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index cbcd91b8..fe4f9bb3 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -65,6 +65,33 @@ module.exports.tests.valid = function(test, common) { t.end(); }); }); + + test('no size, min query size 10', function (t) { + setup(); + const calcSize = calcSizeMiddleware(10); + calcSize(req, {}, function () { + t.equal(req.clean.hasOwnProperty('querySize'), false); + t.end(); + }); + }); + + test('size 5, min query size 10', function (t) { + setup(5); + const calcSize = calcSizeMiddleware(10); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 10); + t.end(); + }); + }); + + test('size 3, min query size 2', function (t) { + setup(3); + const calcSize = calcSizeMiddleware(2); + calcSize(req, {}, function () { + t.equal(req.clean.querySize, 6); + t.end(); + }); + }); }; module.exports.all = function (tape, common) { From f887d7a2499972f7c6da6277df8b615ba53b6fdf Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 12:26:57 -0400 Subject: [PATCH 7/8] Rename test setup function It now can be confused for the middleware setup function --- test/unit/helper/sizeCalculator.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/helper/sizeCalculator.js index fe4f9bb3..204c6870 100644 --- a/test/unit/helper/sizeCalculator.js +++ b/test/unit/helper/sizeCalculator.js @@ -11,7 +11,7 @@ module.exports.tests.interface = function(test, common) { module.exports.tests.valid = function(test, common) { var req = { clean: {} }; - function setup(val) { + function setupQuery(val) { if (isNaN(val)) { delete req.clean.size; } @@ -22,7 +22,7 @@ module.exports.tests.valid = function(test, common) { } test('size=0', function (t) { - setup(0); + setupQuery(0); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -31,7 +31,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=1', function (t) { - setup(1); + setupQuery(1); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -40,7 +40,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=10', function (t) { - setup(10); + setupQuery(10); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 20); @@ -49,7 +49,7 @@ module.exports.tests.valid = function(test, common) { }); test('size=20', function (t) { - setup(20); + setupQuery(20); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.querySize, 40); @@ -58,7 +58,7 @@ module.exports.tests.valid = function(test, common) { }); test('no size', function (t) { - setup(); + setupQuery(); const calcSize = calcSizeMiddleware(); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); @@ -67,7 +67,7 @@ module.exports.tests.valid = function(test, common) { }); test('no size, min query size 10', function (t) { - setup(); + setupQuery(); const calcSize = calcSizeMiddleware(10); calcSize(req, {}, function () { t.equal(req.clean.hasOwnProperty('querySize'), false); @@ -76,7 +76,7 @@ module.exports.tests.valid = function(test, common) { }); test('size 5, min query size 10', function (t) { - setup(5); + setupQuery(5); const calcSize = calcSizeMiddleware(10); calcSize(req, {}, function () { t.equal(req.clean.querySize, 10); @@ -85,7 +85,7 @@ module.exports.tests.valid = function(test, common) { }); test('size 3, min query size 2', function (t) { - setup(3); + setupQuery(3); const calcSize = calcSizeMiddleware(2); calcSize(req, {}, function () { t.equal(req.clean.querySize, 6); From dccad99d78518e727ed6ad7367cee717989d0e0b Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Mar 2018 14:53:10 -0400 Subject: [PATCH 8/8] Move sizeCalculator middleware tests to the correct place --- test/unit/{helper => middleware}/sizeCalculator.js | 0 test/unit/run.js | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/unit/{helper => middleware}/sizeCalculator.js (100%) diff --git a/test/unit/helper/sizeCalculator.js b/test/unit/middleware/sizeCalculator.js similarity index 100% rename from test/unit/helper/sizeCalculator.js rename to test/unit/middleware/sizeCalculator.js diff --git a/test/unit/run.js b/test/unit/run.js index ffebf9ea..1a557733 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -36,7 +36,6 @@ var tests = [ require('./helper/geojsonify'), require('./helper/logging'), require('./helper/type_mapping'), - require('./helper/sizeCalculator'), require('./helper/stackTraceLine'), require('./middleware/access_log'), require('./middleware/accuracy'), @@ -52,6 +51,7 @@ var tests = [ require('./middleware/parseBBox'), require('./middleware/sendJSON'), require('./middleware/normalizeParentIds'), + require('./middleware/sizeCalculator'), require('./middleware/sortResponseData'), require('./middleware/trimByGranularity'), require('./middleware/trimByGranularityStructured'),