Merge pull request #1170 from pelias/trim_text

trim whitespace and quotation marks from a range of natural languages

sanitizer/_text.js

@@ -1,6 +1,9 @@
 const check = require('check-types');
 const _ = require('lodash');
 
+// ref: https://en.wikipedia.org/wiki/Quotation_mark
+const QUOTES = `"'«»‘’‚‛“”„‟‹›⹂「」『』〝〞〟﹁﹂﹃﹄＂＇｢｣`;
+
 // validate texts, convert types and apply defaults
 function _sanitize( raw, clean ){
 
@@ -14,8 +17,7 @@ function _sanitize( raw, clean ){
     messages.errors.push('invalid param \'text\': text length, must be >0');
 
   } else {
-    clean.text = raw.text;
+    clean.text = _.trim( _.trim( raw.text ), QUOTES );
-
   }
 
   return messages;

test/unit/sanitizer/_text.js

@@ -37,7 +37,7 @@ module.exports.tests.text_parser = function(test, common) {
       const messages = sanitizer.sanitize(raw, clean);
 
       t.deepEquals(clean, expected_clean);
-      t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0'], 'no errors');
+      t.deepEquals(messages.errors, ['invalid param \'text\': text length, must be >0']);
       t.deepEquals(messages.warnings, [], 'no warnings');
 
     });
@@ -46,6 +46,78 @@ module.exports.tests.text_parser = function(test, common) {
 
   });
 
+  test('should trim whitespace', t => {
+    var clean = {};
+    var raw = { text: ` test \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  test('should trim double quotes', t => {
+    var clean = {};
+    var raw = { text: ` "test" \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  test('should trim single quotes', t => {
+    var clean = {};
+    var raw = { text: ` 'test' \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  test('should trim German quotes', t => {
+    var clean = {};
+    var raw = { text: ` „test“ \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  test('should trim guillemets', t => {
+    var clean = {};
+    var raw = { text: ` »test« \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
+  test('should trim Chinese quotes', t => {
+    var clean = {};
+    var raw = { text: ` ﹁「test」﹂ \n ` };
+    const messages = sanitizer.sanitize(raw, clean);
+
+    t.equals(clean.text, 'test');
+    t.deepEquals(messages.errors, [], 'no errors');
+    t.deepEquals(messages.warnings, [], 'no warnings');
+
+    t.end();
+  });
+
   test('return an array of expected parameters in object form for validation', (t) => {
     const expected = [{ name: 'text' }];
     const validParameters = sanitizer.expected();