From 5dc9737caa2c01114e0fd2331fd7d691f849be78 Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Thu, 13 Jul 2017 18:05:47 -0400 Subject: [PATCH] removed text-analyzer call from _text sanitizer --- sanitizer/_text.js | 11 -- test/unit/sanitizer/_text.js | 201 ++++------------------------------- 2 files changed, 22 insertions(+), 190 deletions(-) diff --git a/sanitizer/_text.js b/sanitizer/_text.js index d7d1227b..82662473 100644 --- a/sanitizer/_text.js +++ b/sanitizer/_text.js @@ -1,5 +1,4 @@ const check = require('check-types'); -const text_analyzer = require('pelias-text-analyzer'); const _ = require('lodash'); // validate texts, convert types and apply defaults @@ -16,16 +15,6 @@ function sanitize( raw, clean ){ } else { clean.text = raw.text; - // only call libpostal if there are other sources besides whosonfirst - // since placeholder will take care of it later - if (!_.isEqual(clean.sources, ['whosonfirst'])) { - // parse text with query parser - const parsed_text = text_analyzer.parse(clean.text); - if (check.assigned(parsed_text)) { - clean.parsed_text = parsed_text; - } - } - } return messages; diff --git a/test/unit/sanitizer/_text.js b/test/unit/sanitizer/_text.js index a8a46137..78b9a676 100644 --- a/test/unit/sanitizer/_text.js +++ b/test/unit/sanitizer/_text.js @@ -1,213 +1,56 @@ -var type_mapping = require('../../../helper/type_mapping'); -var proxyquire = require('proxyquire').noCallThru(); +const sanitizer = require('../../../sanitizer/_text'); module.exports.tests = {}; module.exports.tests.text_parser = function(test, common) { - test('non-empty raw.text should call analyzer and set clean.text and clean.parsed_text', function(t) { - var mock_analyzer_response = { - key1: 'value 1', - key2: 'value 2' - }; - - var sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - return mock_analyzer_response; - } - }}); - - var raw = { - text: 'raw input' - }; - var clean = { - }; - - var expected_clean = { - text: raw.text, - parsed_text: mock_analyzer_response - }; - - var messages = sanitizer(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('empty raw.text should add error message', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - throw new Error('analyzer should not have been called'); - } - }}); - - var raw = { - text: '' - }; - var clean = { - }; - - var expected_clean = { - }; - - var messages = sanitizer(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('undefined raw.text should add error message', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - throw new Error('analyzer should not have been called'); - } - }}); - - var raw = { - text: undefined - }; - var clean = { - }; - - var expected_clean = { - }; - - var messages = sanitizer(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('text_analyzer.parse returning undefined should not overwrite clean.parsed_text', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - return undefined; - } - }}); - - var raw = { - text: 'raw input' - }; - var clean = { - parsed_text: 'original clean.parsed_text' - }; - - var expected_clean = { - text: raw.text, - parsed_text: 'original clean.parsed_text' - }; - - var messages = sanitizer(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('text_analyzer.parse returning null should not overwrite clean.parsed_text', function(t) { - var sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - return null; - } - }}); - - var raw = { - text: 'raw input' - }; - var clean = { - parsed_text: 'original clean.parsed_text' - }; - - var expected_clean = { - text: raw.text, - parsed_text: 'original clean.parsed_text' - }; - - var messages = sanitizer(raw, clean); - - t.deepEquals(clean, expected_clean); - t.deepEquals(messages.errors, [], 'no errors'); - t.deepEquals(messages.warnings, [], 'no warnings'); - t.end(); - - }); - - test('sources=whosonfirst should not call text_analyzer and set clean.text from raw.text', (t) => { - const sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: query => t.fail('should not have been called') } - }); - + test('non-empty raw.text should call analyzer and set clean.text and not clean.parsed_text', t => { const raw = { - text: 'raw clean.text' + text: 'raw input' }; const clean = { - sources: ['whosonfirst'], text: 'original clean.text' }; const expected_clean = { - sources: ['whosonfirst'], - text: 'raw clean.text' + text: raw.text }; const messages = sanitizer(raw, clean); t.deepEquals(clean, expected_clean); - t.deepEquals(messages, { errors: [], warnings: [] }); + t.deepEquals(messages, { warnings: [], errors: [] }, 'no errors/warnings'); t.end(); }); - test('sources with whosonfirst + others should call analyzer', (t) => { - const sanitizer = proxyquire('../../../sanitizer/_text', { - 'pelias-text-analyzer': { parse: function(query) { - return { - key1: 'value 1', - key2: 'value 2' - }; - } - }}); + test('undefined/empty raw.text should add error message', t => { + [undefined, ''].forEach(val => { + const raw = { + text: val + }; + const clean = { + }; - const raw = { - text: 'raw text' - }; - const clean = { - sources: ['whosonfirst', 'another source'], - text: 'clean text' - }; + const expected_clean = { + }; - const expected_clean = { - sources: ['whosonfirst', 'another source'], - text: 'raw text', - parsed_text: { - key1: 'value 1', - key2: 'value 2' - } - }; + const messages = sanitizer(raw, clean); - const messages = sanitizer(raw, clean); + t.deepEquals(clean, expected_clean); + t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors'); + t.deepEquals(messages.warnings, [], 'no warnings'); + + }); - t.deepEquals(clean, expected_clean); - t.deepEquals(messages, { errors: [], warnings: [] }); t.end(); }); }; -module.exports.all = function (tape, common) { +module.exports.all = (tape, common) => { function test(name, testFunction) { - return tape('sanitizeR _text: ' + name, testFunction); + return tape(`sanitizer _text: ${name}`, testFunction); } for( var testCase in module.exports.tests ){