From fd529ccee9ae95073df92d3850a289b55169252a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 14 Apr 2016 15:50:48 -0400 Subject: [PATCH 1/8] Use new express sendStatus method According to a message on the console: ``` express deprecated res.send(status): Use res.sendStatus(status) instead middleware/options.js:12:9 ``` --- middleware/options.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/options.js b/middleware/options.js index 57c0e1eb..f6ca8767 100644 --- a/middleware/options.js +++ b/middleware/options.js @@ -9,10 +9,10 @@ function middleware(req, res, next){ if( req.method === 'OPTIONS' ){ - res.send(200); + res.sendStatus(200); } else { next(); } } -module.exports = middleware; \ No newline at end of file +module.exports = middleware; From 7876650581922c3c6b24794b2c940afcc5cd9619 Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Fri, 15 Apr 2016 14:28:04 -0700 Subject: [PATCH 2/8] chore(package): update pelias-model to version 4.0.0 http://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0c0cc1f7..ed536057 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "morgan": "1.7.0", "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", - "pelias-model": "^3.1.0", + "pelias-model": "^4.0.0", "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", "stats-lite": "1.0.3", From 2c4082770153c442585272a63b25ba547945c3fd Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 15 Apr 2016 20:42:38 -0400 Subject: [PATCH 3/8] added `layer=borough` support for *all 5 boroughs* in data --- helper/type_mapping.js | 3 ++- test/ciao/autocomplete/layers_alias_coarse.coffee | 1 + test/ciao/autocomplete/layers_invalid.coffee | 2 +- test/ciao/autocomplete/layers_mix_invalid_valid.coffee | 2 +- test/ciao/reverse/layers_alias_coarse.coffee | 1 + test/ciao/reverse/layers_invalid.coffee | 2 +- test/ciao/reverse/layers_mix_invalid_valid.coffee | 2 +- test/ciao/search/layers_alias_coarse.coffee | 1 + test/ciao/search/layers_invalid.coffee | 2 +- test/ciao/search/layers_mix_invalid_valid.coffee | 2 +- test/unit/helper/type_mapping.js | 2 +- test/unit/sanitiser/_layers.js | 10 +++++----- 12 files changed, 17 insertions(+), 13 deletions(-) diff --git a/helper/type_mapping.js b/helper/type_mapping.js index 0b20c111..ed20c0d8 100644 --- a/helper/type_mapping.js +++ b/helper/type_mapping.js @@ -49,7 +49,8 @@ var LAYERS_BY_SOURCE = { openaddresses: [ 'address' ], geonames: [ 'country', 'region', 'county', 'locality', 'venue' ], whosonfirst: [ 'continent', 'country', 'dependency', 'macroregion', 'region', - 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'neighbourhood', 'microhood', 'disputed'] + 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'borough', + 'neighbourhood', 'microhood', 'disputed'] }; /* diff --git a/test/ciao/autocomplete/layers_alias_coarse.coffee b/test/ciao/autocomplete/layers_alias_coarse.coffee index 2fa2265c..3db308be 100644 --- a/test/ciao/autocomplete/layers_alias_coarse.coffee +++ b/test/ciao/autocomplete/layers_alias_coarse.coffee @@ -41,6 +41,7 @@ json.geocoding.query.layers.should.eql [ "continent", "macrocounty", "county", "macrohood", + "borough", "neighbourhood", "microhood", "disputed" diff --git a/test/ciao/autocomplete/layers_invalid.coffee b/test/ciao/autocomplete/layers_invalid.coffee index 620b5586..6f3cebe0 100644 --- a/test/ciao/autocomplete/layers_invalid.coffee +++ b/test/ciao/autocomplete/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/autocomplete/layers_mix_invalid_valid.coffee b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee index 963b79ab..a819dd44 100644 --- a/test/ciao/autocomplete/layers_mix_invalid_valid.coffee +++ b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_alias_coarse.coffee b/test/ciao/reverse/layers_alias_coarse.coffee index 09c91483..40ce2e37 100644 --- a/test/ciao/reverse/layers_alias_coarse.coffee +++ b/test/ciao/reverse/layers_alias_coarse.coffee @@ -40,6 +40,7 @@ json.geocoding.query.layers.should.eql [ "continent", "macrocounty", "county", "macrohood", + "borough", "neighbourhood", "microhood", "disputed" diff --git a/test/ciao/reverse/layers_invalid.coffee b/test/ciao/reverse/layers_invalid.coffee index aaec4864..bc57a3b3 100644 --- a/test/ciao/reverse/layers_invalid.coffee +++ b/test/ciao/reverse/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/reverse/layers_mix_invalid_valid.coffee b/test/ciao/reverse/layers_mix_invalid_valid.coffee index 307b225d..16f40b9e 100644 --- a/test/ciao/reverse/layers_mix_invalid_valid.coffee +++ b/test/ciao/reverse/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_alias_coarse.coffee b/test/ciao/search/layers_alias_coarse.coffee index bf7cdb52..48723853 100644 --- a/test/ciao/search/layers_alias_coarse.coffee +++ b/test/ciao/search/layers_alias_coarse.coffee @@ -41,6 +41,7 @@ json.geocoding.query.layers.should.eql [ "continent", "macrocounty", "county", "macrohood", + "borough", "neighbourhood", "microhood", "disputed" diff --git a/test/ciao/search/layers_invalid.coffee b/test/ciao/search/layers_invalid.coffee index 4f2da456..cc6feab8 100644 --- a/test/ciao/search/layers_invalid.coffee +++ b/test/ciao/search/layers_invalid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/ciao/search/layers_mix_invalid_valid.coffee b/test/ciao/search/layers_mix_invalid_valid.coffee index f004c69e..f332e051 100644 --- a/test/ciao/search/layers_mix_invalid_valid.coffee +++ b/test/ciao/search/layers_mix_invalid_valid.coffee @@ -24,7 +24,7 @@ json.features.should.be.instanceof Array #? expected errors should.exist json.geocoding.errors -json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,macroregion,region,county,locality,continent,macrocounty,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ] +json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,dependency,macroregion,localadmin,macrocounty,macrohood,borough,neighbourhood,microhood,disputed' ] #? expected warnings should.not.exist json.geocoding.warnings diff --git a/test/unit/helper/type_mapping.js b/test/unit/helper/type_mapping.js index 355fd4e6..a9ec4721 100644 --- a/test/unit/helper/type_mapping.js +++ b/test/unit/helper/type_mapping.js @@ -14,7 +14,7 @@ module.exports.tests.interfaces = function(test, common) { t.deepEquals(type_mapping.layer_mapping.coarse, [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', - 'neighbourhood', 'microhood', 'disputed' ]); + 'borough', 'neighbourhood', 'microhood', 'disputed' ]); t.end(); }); diff --git a/test/unit/sanitiser/_layers.js b/test/unit/sanitiser/_layers.js index b9dcbd0f..a1fde0f4 100644 --- a/test/unit/sanitiser/_layers.js +++ b/test/unit/sanitiser/_layers.js @@ -42,8 +42,8 @@ module.exports.tests.sanitize_layers = function(test, common) { sanitize(raw, clean); var admin_layers = [ 'continent', 'country', 'dependency', - 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'neighbourhood', - 'microhood', 'disputed' ]; + 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', + 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ]; t.deepEqual(clean.layers, admin_layers, 'coarse layers set'); t.end(); @@ -77,8 +77,8 @@ module.exports.tests.sanitize_layers = function(test, common) { sanitize(raw, clean); var expected_layers = [ 'continent', 'country', 'dependency', - 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', 'macrohood', 'neighbourhood', - 'microhood', 'disputed' ]; + 'macroregion', 'region', 'locality', 'localadmin', 'macrocounty', 'county', + 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ]; t.deepEqual(clean.layers, expected_layers, 'coarse + regular layers set'); t.end(); @@ -114,7 +114,7 @@ module.exports.tests.sanitize_layers = function(test, common) { var coarse_layers = [ 'continent', 'country', 'dependency', 'macroregion', 'region', 'locality', 'localadmin', - 'macrocounty', 'county', 'macrohood', 'neighbourhood', 'microhood', + 'macrocounty', 'county', 'macrohood', 'borough', 'neighbourhood', 'microhood', 'disputed' ]; var venue_layers = [ 'venue' ]; From 9cdfce13c18955aab771840fee524d3e2ff64c2a Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Mon, 18 Apr 2016 15:17:08 -0400 Subject: [PATCH 4/8] fixed extraneous comma labels for results with only a `name` --- helper/labelGenerator.js | 27 ++++++++++++++------- test/unit/helper/labelGenerator_examples.js | 11 +++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/helper/labelGenerator.js b/helper/labelGenerator.js index 20c73a98..97993e03 100644 --- a/helper/labelGenerator.js +++ b/helper/labelGenerator.js @@ -16,18 +16,27 @@ module.exports = function( record ){ // retain only things that are truthy labelParts = _.compact(labelParts); - // first, dedupe the name and 1st label array elements - // this is used to ensure that the `name` and first admin hierarchy elements aren't repeated - // eg - `["Lancaster", "Lancaster", "PA", "United States"]` -> `["Lancaster", "PA", "United States"]` - var dedupedNameAndFirstLabelElement = _.uniq([labelParts.shift(), labelParts.shift()]); + // third, dedupe and join with a comma and return + return dedupeNameAndFirstLabelElement(labelParts).join(', '); - // second, unshift the deduped parts back onto the labelParts - labelParts.unshift.apply(labelParts, dedupedNameAndFirstLabelElement); +}; - // third, join with a comma and return - return labelParts.join(', '); +function dedupeNameAndFirstLabelElement(labelParts) { + // only dedupe if a result has more than a name (the first label part) + if (labelParts.length > 1) { + // first, dedupe the name and 1st label array elements + // this is used to ensure that the `name` and first admin hierarchy elements aren't repeated + // eg - `["Lancaster", "Lancaster", "PA", "United States"]` -> `["Lancaster", "PA", "United States"]` + var deduped = _.uniq([labelParts.shift(), labelParts.shift()]); -}; + // second, unshift the deduped parts back onto the labelParts + labelParts.unshift.apply(labelParts, deduped); + + } + + return labelParts; + +} function getSchema(country_a) { if (country_a && country_a.length && schemas[country_a]) { diff --git a/test/unit/helper/labelGenerator_examples.js b/test/unit/helper/labelGenerator_examples.js index 416b7598..3207df86 100644 --- a/test/unit/helper/labelGenerator_examples.js +++ b/test/unit/helper/labelGenerator_examples.js @@ -104,6 +104,17 @@ module.exports.tests.france = function(test, common) { }; +module.exports.tests.name_only = function(test, common) { + test('name-only results (no admin fields) should not include extraneous comma', function(t) { + var doc = { + 'name': 'Result name', + }; + t.equal(generator(doc),'Result name'); + t.end(); + }); + +}; + module.exports.all = function (tape, common) { function test(name, testFunction) { From bd3b8e7bbed7acdf1b4266cc122c66390e818b0b Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Tue, 19 Apr 2016 21:38:56 -0400 Subject: [PATCH 5/8] switched to pelias-text-analyzer package since that responsibility has been moved to there --- helper/text_parser.js | 86 ------------------ package.json | 2 +- sanitiser/_text.js | 4 +- test/unit/helper/text_parser.js | 150 -------------------------------- test/unit/query/autocomplete.js | 1 - test/unit/query/search.js | 8 +- test/unit/run.js | 1 - test/unit/sanitiser/search.js | 4 +- 8 files changed, 9 insertions(+), 247 deletions(-) delete mode 100644 helper/text_parser.js delete mode 100644 test/unit/helper/text_parser.js diff --git a/helper/text_parser.js b/helper/text_parser.js deleted file mode 100644 index 0db8bede..00000000 --- a/helper/text_parser.js +++ /dev/null @@ -1,86 +0,0 @@ - -var parser = require('addressit'); -var extend = require('extend'); -var type_mapping = require('../helper/type_mapping'); -var check = require('check-types'); -var logger = require('pelias-logger').get('api'); - -var DELIM = ','; - -/* - * For performance, and to prefer POI and admin records, express a preference - * to only search coarse layers on very short text inputs. - */ -module.exports.get_layers = function get_layers(query) { - if (query.length <= 3 ) { - // no address parsing required - return type_mapping.layer_mapping.coarse; - } -}; - -module.exports.get_parsed_address = function get_parsed_address(query) { - - var getAdminPartsBySplittingOnDelim = function(queryParts) { - // naive approach - for admin matching during query time - // split 'flatiron, new york, ny' into 'flatiron' and 'new york, ny' - - var address = {}; - - if (queryParts.length > 1) { - address.name = queryParts[0].trim(); - - // 1. slice away all parts after the first one - // 2. trim spaces from each part just in case - // 3. join the parts back together with appropriate delimiter and spacing - address.admin_parts = queryParts.slice(1) - .map(function (part) { return part.trim(); }) - .join(DELIM + ' '); - } - - return address; - }; - - var getAddressParts = function(query) { - // perform full address parsing - // except on queries so short they obviously can't contain an address - if (query.length > 3) { - return parser( query ); - } - }; - - var queryParts = query.split(DELIM); - - var addressWithAdminParts = getAdminPartsBySplittingOnDelim(queryParts); - var addressWithAddressParts= getAddressParts(queryParts.join(DELIM + ' ')); - - var parsedAddress = extend(addressWithAdminParts, - addressWithAddressParts); - - var address_parts = [ 'name', - 'number', - 'street', - 'city', - 'state', - 'country', - 'postalcode', - 'regions', - 'admin_parts' - ]; - - var parsed_text = {}; - - address_parts.forEach(function(part){ - if (parsedAddress[part]) { - parsed_text[part] = parsedAddress[part]; - } - }); - - // if all we found was regions, ignore it as it is not enough information to make smarter decisions - if (Object.keys(parsed_text).length === 1 && !check.undefined(parsed_text.regions)) - { - logger.info('Ignoring address parser output, regions only'); - return null; - } - - return parsed_text; -}; diff --git a/package.json b/package.json index 0c0cc1f7..fc511657 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "elasticsearch": ">=1.2.1" }, "dependencies": { - "addressit": "git://github.com/dianashk/addressit.git#temp", "async": "^1.5.2", "check-types": "^6.0.0", "cluster2": "git://github.com/missinglink/cluster2.git#node_zero_twelve", @@ -56,6 +55,7 @@ "pelias-model": "^3.1.0", "pelias-query": "6.2.0", "pelias-suggester-pipeline": "2.0.4", + "pelias-text-analyzer": "^1.0.1", "stats-lite": "1.0.3", "through2": "2.0.1" }, diff --git a/sanitiser/_text.js b/sanitiser/_text.js index e6897a5e..4709eeee 100644 --- a/sanitiser/_text.js +++ b/sanitiser/_text.js @@ -1,5 +1,5 @@ var check = require('check-types'), - text_parser = require('../helper/text_parser'); + text_analyzer = require('pelias-text-analyzer'); // validate texts, convert types and apply defaults function sanitize( raw, clean ){ @@ -19,7 +19,7 @@ function sanitize( raw, clean ){ clean.text = raw.text; // parse text with query parser - var parsed_text = text_parser.get_parsed_address(clean.text); + var parsed_text = text_analyzer.parse(clean.text); if (check.assigned(parsed_text)) { clean.parsed_text = parsed_text; } diff --git a/test/unit/helper/text_parser.js b/test/unit/helper/text_parser.js deleted file mode 100644 index ca5b05f0..00000000 --- a/test/unit/helper/text_parser.js +++ /dev/null @@ -1,150 +0,0 @@ -var parser = require('../../../helper/text_parser'); - -var type_mapping = require('../../../helper/type_mapping'); -var layers_map = type_mapping.layer_mapping; - -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('interface', function(t) { - t.equal(typeof parser.get_parsed_address, 'function', 'valid function'); - t.equal(typeof parser.get_layers, 'function', 'valid function'); - t.end(); - }); -}; - -module.exports.tests.split_on_comma = function(test, common) { - var queries = [ - { name: 'soho', admin_parts: 'new york' }, - { name: 'chelsea', admin_parts: 'london' }, - { name: '123 main', admin_parts: 'new york' } - ]; - - queries.forEach(function (query) { - test('naive parsing ' + query, function(t) { - var address = parser.get_parsed_address(query.name + ', ' + query.admin_parts); - - t.equal(typeof address, 'object', 'valid object'); - t.equal(address.name, query.name, 'name set correctly to ' + address.name); - t.equal(address.admin_parts, query.admin_parts, 'admin_parts set correctly to ' + address.admin_parts); - t.end(); - }); - - test('naive parsing ' + query + 'without spaces', function(t) { - var address = parser.get_parsed_address(query.name + ',' + query.admin_parts); - - t.equal(typeof address, 'object', 'valid object'); - t.equal(address.name, query.name, 'name set correctly to ' + address.name); - t.equal(address.admin_parts, query.admin_parts, 'admin_parts set correctly to ' + address.admin_parts); - t.end(); - }); - }); -}; - -module.exports.tests.parse_three_chars_or_less = function(test, common) { - var chars_queries = ['a', 'bb', 'ccc']; - var num_queries = ['1', '12', '123']; - var alphanum_q = ['a1', '1a2', '12c']; - - var queries = chars_queries.concat(num_queries).concat(alphanum_q); - queries.forEach(function(query) { - test('query length < 3 (' + query + ')', function(t) { - var address = parser.get_parsed_address(query); - var target_layer = layers_map.coarse; - var layers = parser.get_layers(query); - - t.equal(typeof address, 'object', 'valid object'); - t.deepEqual(layers, target_layer, 'admin_parts set correctly to ' + target_layer.join(', ')); - t.end(); - }); - }); -}; - -module.exports.tests.parse_one_token = function(test, common) { - test('query with one token', function (t) { - var address = parser.get_parsed_address('yugolsavia'); - t.equal(address, null, 'nothing address specific detected'); - t.end(); - }); - test('query with two tokens, no numbers', function (t) { - var address = parser.get_parsed_address('small town'); - t.equal(address, null, 'nothing address specific detected'); - t.end(); - }); - test('query with two tokens, number first', function (t) { - var address = parser.get_parsed_address('123 main'); - t.equal(address, null, 'nothing address specific detected'); - t.end(); - }); - test('query with two tokens, number second', function (t) { - var address = parser.get_parsed_address('main 123'); - t.equal(address, null, 'nothing address specific detected'); - t.end(); - }); - test('query with many tokens', function(t) { - var address = parser.get_parsed_address('main particle new york'); - t.equal(address, null, 'nothing address specific detected'); - t.end(); - }); -}; - -module.exports.tests.parse_address = function(test, common) { - test('valid address, house number', function(t) { - var query_string = '123 main st new york ny'; - var address = parser.get_parsed_address(query_string); - - t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.number, '123', 'parsed house number'); - t.equal(address.street, 'main st', 'parsed street'); - t.deepEqual(address.regions, ['new york'], 'parsed city'); - t.equal(address.state , 'NY', 'parsed state'); - t.end(); - }); - test('valid address, zipcode', function(t) { - var query_string = '123 main st new york ny 10010'; - var address = parser.get_parsed_address(query_string); - - t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.number, '123', 'parsed house number'); - t.equal(address.street, 'main st', 'parsed street'); - t.deepEqual(address.regions, ['new york'], 'parsed city'); - t.equal(address.state , 'NY', 'parsed state'); - t.equal(address.postalcode, '10010', 'parsed zip is a string'); - t.end(); - }); - test('valid address with leading 0s in zipcode', function(t) { - var query_string = '339 W Main St, Cheshire, 06410'; - var address = parser.get_parsed_address(query_string); - - console.log(address); - - t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.street, 'W Main St', 'parsed street'); - t.deepEqual(address.regions, ['Cheshire'], 'parsed city'); - t.equal(address.postalcode, '06410', 'parsed zip'); - t.end(); - }); - test('valid address without spaces after commas', function(t) { - var query_string = '339 W Main St,Lancaster,PA'; - var address = parser.get_parsed_address(query_string); - - t.equal(typeof address, 'object', 'valid object for the address'); - t.equal(address.number, '339', 'parsed house number'); - t.equal(address.street, 'W Main St', 'parsed street'); - t.deepEqual(address.regions, ['Lancaster'], 'parsed city'); - t.deepEqual(address.state, 'PA', 'parsed state'); - t.end(); - }); -}; - - -module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('QUERY PARSING: ' + name, testFunction); - } - - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); - } -}; diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js index 0e09457b..f4b44ffa 100644 --- a/test/unit/query/autocomplete.js +++ b/test/unit/query/autocomplete.js @@ -1,6 +1,5 @@ var generate = require('../../../query/autocomplete'); -var parser = require('../../../helper/text_parser'); module.exports.tests = {}; diff --git a/test/unit/query/search.js b/test/unit/query/search.js index e503311b..426eb2bc 100644 --- a/test/unit/query/search.js +++ b/test/unit/query/search.js @@ -1,5 +1,5 @@ var generate = require('../../../query/search'); -var parser = require('../../../helper/text_parser'); +var text_analyzer = require('pelias-text-analyzer'); module.exports.tests = {}; @@ -128,7 +128,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: address, layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, - parsed_text: parser.get_parsed_address(address), + parsed_text: text_analyzer.parse(address), }); var compiled = JSON.parse( JSON.stringify( query ) ); @@ -143,7 +143,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, - parsed_text: parser.get_parsed_address(partial_address), + parsed_text: text_analyzer.parse(partial_address), }); var compiled = JSON.parse( JSON.stringify( query ) ); @@ -158,7 +158,7 @@ module.exports.tests.query = function(test, common) { var query = generate({ text: partial_address, layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], querySize: 10, - parsed_text: parser.get_parsed_address(partial_address), + parsed_text: text_analyzer.parse(partial_address), }); var compiled = JSON.parse( JSON.stringify( query ) ); diff --git a/test/unit/run.js b/test/unit/run.js index 6f64f2e6..94d9ebb3 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -19,7 +19,6 @@ var tests = [ require('./helper/labelGenerator_GBR'), require('./helper/labelGenerator_USA'), require('./helper/labelSchema'), - require('./helper/text_parser'), require('./helper/type_mapping'), require('./helper/sizeCalculator'), require('./middleware/confidenceScore'), diff --git a/test/unit/sanitiser/search.js b/test/unit/sanitiser/search.js index e09672ce..864195a8 100644 --- a/test/unit/sanitiser/search.js +++ b/test/unit/sanitiser/search.js @@ -1,6 +1,6 @@ var extend = require('extend'), search = require('../../../sanitiser/search'), - parser = require('../../../helper/text_parser'), + text_analyzer = require('pelias-text-analyzer'), sanitize = search.sanitize, middleware = search.middleware, defaultError = 'invalid param \'text\': text length, must be >0'; @@ -80,7 +80,7 @@ module.exports.tests.sanitize_text_with_delim = function(test, common) { sanitize( req, function( ){ var expected_text = text; - var expected_parsed_text = parser.get_parsed_address(text); + var expected_parsed_text = text_analyzer.parse(text); t.equal(req.errors[0], undefined, 'no error'); t.equal(req.clean.parsed_text.name, expected_parsed_text.name, 'clean name set correctly'); t.equal(req.clean.text, expected_text, 'text should match'); From 028ab1d8898c29e2dbfb8c642cf1bf7431d9cd10 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 21 Apr 2016 14:58:01 -0400 Subject: [PATCH 6/8] Add search config settings for borough --- query/search_defaults.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/query/search_defaults.js b/query/search_defaults.js index ea0dc87f..89aca7d6 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -78,6 +78,10 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'admin:locality:field': 'parent.locality', 'admin:locality:boost': 1, + 'admin:borough:analyzer': 'peliasAdmin', + 'admin:borough:field': 'parent.borough', + 'admin:borough:boost': 1, + 'admin:neighbourhood:analyzer': 'peliasAdmin', 'admin:neighbourhood:field': 'parent.neighbourhood', 'admin:neighbourhood:boost': 1, From f8af354e71f06cc5deb65e5285848d60f442db58 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 21 Apr 2016 14:58:17 -0400 Subject: [PATCH 7/8] Base admin fields on placetypes --- query/text_parser.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/query/text_parser.js b/query/text_parser.js index 38fca48e..d19465eb 100644 --- a/query/text_parser.js +++ b/query/text_parser.js @@ -1,20 +1,15 @@ var logger = require('pelias-logger').get('api'); +var placeTypes = require('../helper/placeTypes'); /* This list should only contain admin fields we are comfortable matching in the case when we can't identify parts of an address. This shouldn't contain fields like country_a or postalcode because we should only try to match those when we're sure that's what they are. */ -var adminFields = [ - 'country', - 'region', +var adminFields = placeTypes.concat([ 'region_a', - 'county', - 'localadmin', - 'locality', - 'neighbourhood' -]; +]); /** @todo: refactor me @@ -101,4 +96,4 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ } } -module.exports = addParsedVariablesToQueryVariables; \ No newline at end of file +module.exports = addParsedVariablesToQueryVariables; From bf3d931640d0747090257fdbf5793456acc2b26f Mon Sep 17 00:00:00 2001 From: greenkeeperio-bot Date: Thu, 21 Apr 2016 14:46:22 -0700 Subject: [PATCH 8/8] chore(package): update pelias-query to version 6.3.0 https://greenkeeper.io/ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a6e5c30d..8e325f5d 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "pelias-config": "^1.0.1", "pelias-logger": "^0.0.8", "pelias-model": "^4.0.0", - "pelias-query": "6.2.0", + "pelias-query": "6.3.0", "pelias-suggester-pipeline": "2.0.4", "pelias-text-analyzer": "^1.0.1", "stats-lite": "1.0.3",