From ed0a96cff20ba459f62ef820a380c393306348b3 Mon Sep 17 00:00:00 2001 From: Horace Williams Date: Fri, 3 Aug 2018 17:20:10 -0700 Subject: [PATCH 01/17] Migrate to internal iso3166 defnitions This is a somewhat roundabout fix to #1179, as a way to deal with the persistent npm ls and commit hook troubles we were dealing with due to dependencies of the iso3166 package. Additionally it should give us a faster definition of these ISO lookups, since the existing approaches were implemented using linear scans through an array rather than map-based lookups. --- controller/libpostal.js | 6 +- helper/iso3166.js | 324 +++++++++++++++++++++++++++++++++ package.json | 1 - sanitizer/_boundary_country.js | 8 +- sanitizer/_iso2_to_iso3.js | 6 +- test/unit/helper/iso3166.js | 67 +++++++ test/unit/run.js | 1 + 7 files changed, 402 insertions(+), 11 deletions(-) create mode 100644 helper/iso3166.js create mode 100644 test/unit/helper/iso3166.js diff --git a/controller/libpostal.js b/controller/libpostal.js index 7ecbe3b6..dd85e9a2 100644 --- a/controller/libpostal.js +++ b/controller/libpostal.js @@ -1,5 +1,5 @@ const _ = require('lodash'); -const iso3166 = require('iso3166-1'); +const iso3166 = require('../helper/iso3166'); const Debug = require('../helper/debug'); const debugLog = new Debug('controller:libpostal'); const logger = require('pelias-logger').get('api'); @@ -91,8 +91,8 @@ function setup(libpostalService, should_execute) { return o; }, {}); - if (_.has(req.clean.parsed_text, 'country') && iso3166.is2(_.toUpper(req.clean.parsed_text.country))) { - req.clean.parsed_text.country = iso3166.to3(_.toUpper(req.clean.parsed_text.country)); + if (_.has(req.clean.parsed_text, 'country') && iso3166.isISO2Code(req.clean.parsed_text.country)) { + req.clean.parsed_text.country = iso3166.convertISO2ToISO3(req.clean.parsed_text.country); } debugLog.push(req, {parsed_text: req.clean.parsed_text}); diff --git a/helper/iso3166.js b/helper/iso3166.js new file mode 100644 index 00000000..d07da32d --- /dev/null +++ b/helper/iso3166.js @@ -0,0 +1,324 @@ +const _ = require('lodash'); + +const byISO2 = {'AF':{'country':'Afghanistan','alpha2':'AF','alpha3':'AFG','numeric':'004'}, + 'AX':{'country':'Åland Islands','alpha2':'AX','alpha3':'ALA','numeric':'248'}, + 'AL':{'country':'Albania','alpha2':'AL','alpha3':'ALB','numeric':'008'}, + 'DZ':{'country':'Algeria','alpha2':'DZ','alpha3':'DZA','numeric':'012'}, + 'AS':{'country':'American Samoa','alpha2':'AS','alpha3':'ASM','numeric':'016'}, + 'AD':{'country':'Andorra','alpha2':'AD','alpha3':'AND','numeric':'020'}, + 'AO':{'country':'Angola','alpha2':'AO','alpha3':'AGO','numeric':'024'}, + 'AI':{'country':'Anguilla','alpha2':'AI','alpha3':'AIA','numeric':'660'}, + 'AQ':{'country':'Antarctica','alpha2':'AQ','alpha3':'ATA','numeric':'010'}, + 'AG':{'country':'Antigua and Barbuda','alpha2':'AG','alpha3':'ATG','numeric':'028'}, + 'AR':{'country':'Argentina','alpha2':'AR','alpha3':'ARG','numeric':'032'}, + 'AM':{'country':'Armenia','alpha2':'AM','alpha3':'ARM','numeric':'051'}, + 'AW':{'country':'Aruba','alpha2':'AW','alpha3':'ABW','numeric':'533'}, + 'AU':{'country':'Australia','alpha2':'AU','alpha3':'AUS','numeric':'036'}, + 'AT':{'country':'Austria','alpha2':'AT','alpha3':'AUT','numeric':'040'}, + 'AZ':{'country':'Azerbaijan','alpha2':'AZ','alpha3':'AZE','numeric':'031'}, + 'BS':{'country':'Bahamas','alpha2':'BS','alpha3':'BHS','numeric':'044'}, + 'BH':{'country':'Bahrain','alpha2':'BH','alpha3':'BHR','numeric':'048'}, + 'BD':{'country':'Bangladesh','alpha2':'BD','alpha3':'BGD','numeric':'050'}, + 'BB':{'country':'Barbados','alpha2':'BB','alpha3':'BRB','numeric':'052'}, + 'BY':{'country':'Belarus','alpha2':'BY','alpha3':'BLR','numeric':'112'}, + 'BE':{'country':'Belgium','alpha2':'BE','alpha3':'BEL','numeric':'056'}, + 'BZ':{'country':'Belize','alpha2':'BZ','alpha3':'BLZ','numeric':'084'}, + 'BJ':{'country':'Benin','alpha2':'BJ','alpha3':'BEN','numeric':'204'}, + 'BM':{'country':'Bermuda','alpha2':'BM','alpha3':'BMU','numeric':'060'}, + 'BT':{'country':'Bhutan','alpha2':'BT','alpha3':'BTN','numeric':'064'}, + 'BO':{'country':'Bolivia','alpha2':'BO','alpha3':'BOL','numeric':'068'}, + 'BQ':{'country':'Bonaire, Sint Eustatius and Saba','alpha2':'BQ','alpha3':'BES','numeric':'535'}, + 'BA':{'country':'Bosnia and Herzegovina','alpha2':'BA','alpha3':'BIH','numeric':'070'}, + 'BW':{'country':'Botswana','alpha2':'BW','alpha3':'BWA','numeric':'072'}, + 'BV':{'country':'Bouvet Island','alpha2':'BV','alpha3':'BVT','numeric':'074'}, + 'BR':{'country':'Brazil','alpha2':'BR','alpha3':'BRA','numeric':'076'}, + 'IO':{'country':'British Indian Ocean Territory','alpha2':'IO','alpha3':'IOT','numeric':'086'}, + 'BN':{'country':'Brunei Darussalam','alpha2':'BN','alpha3':'BRN','numeric':'096'}, + 'BG':{'country':'Bulgaria','alpha2':'BG','alpha3':'BGR','numeric':'100'}, + 'BF':{'country':'Burkina Faso','alpha2':'BF','alpha3':'BFA','numeric':'854'}, + 'BI':{'country':'Burundi','alpha2':'BI','alpha3':'BDI','numeric':'108'}, + 'CV':{'country':'Cabo Verde','alpha2':'CV','alpha3':'CPV','numeric':'132'}, + 'KH':{'country':'Cambodia','alpha2':'KH','alpha3':'KHM','numeric':'116'}, + 'CM':{'country':'Cameroon','alpha2':'CM','alpha3':'CMR','numeric':'120'}, + 'CA':{'country':'Canada','alpha2':'CA','alpha3':'CAN','numeric':'124'}, + 'KY':{'country':'Cayman Islands','alpha2':'KY','alpha3':'CYM','numeric':'136'}, + 'CF':{'country':'Central African Republic','alpha2':'CF','alpha3':'CAF','numeric':'140'}, + 'TD':{'country':'Chad','alpha2':'TD','alpha3':'TCD','numeric':'148'}, + 'CL':{'country':'Chile','alpha2':'CL','alpha3':'CHL','numeric':'152'}, + 'CN':{'country':'China','alpha2':'CN','alpha3':'CHN','numeric':'156'}, + 'CX':{'country':'Christmas Island','alpha2':'CX','alpha3':'CXR','numeric':'162'}, + 'CC':{'country':'Cocos Islands','alpha2':'CC','alpha3':'CCK','numeric':'166'}, + 'CO':{'country':'Colombia','alpha2':'CO','alpha3':'COL','numeric':'170'}, + 'KM':{'country':'Comoros','alpha2':'KM','alpha3':'COM','numeric':'174'}, + 'CG':{'country':'Congo','alpha2':'CG','alpha3':'COG','numeric':'178'}, + 'CD':{'country':'Congo','alpha2':'CD','alpha3':'COD','numeric':'180'}, + 'CK':{'country':'Cook Islands','alpha2':'CK','alpha3':'COK','numeric':'184'}, + 'CR':{'country':'Costa Rica','alpha2':'CR','alpha3':'CRI','numeric':'188'}, + 'CI':{'country':'Côte d\'Ivoire','alpha2':'CI','alpha3':'CIV','numeric':'384'}, + 'HR':{'country':'Croatia','alpha2':'HR','alpha3':'HRV','numeric':'191'}, + 'CU':{'country':'Cuba','alpha2':'CU','alpha3':'CUB','numeric':'192'}, + 'CW':{'country':'Curaçao','alpha2':'CW','alpha3':'CUW','numeric':'531'}, + 'CY':{'country':'Cyprus','alpha2':'CY','alpha3':'CYP','numeric':'196'}, + 'CZ':{'country':'Czech Republic','alpha2':'CZ','alpha3':'CZE','numeric':'203'}, + 'DK':{'country':'Denmark','alpha2':'DK','alpha3':'DNK','numeric':'208'}, + 'DJ':{'country':'Djibouti','alpha2':'DJ','alpha3':'DJI','numeric':'262'}, + 'DM':{'country':'Dominica','alpha2':'DM','alpha3':'DMA','numeric':'212'}, + 'DO':{'country':'Dominican Republic','alpha2':'DO','alpha3':'DOM','numeric':'214'}, + 'EC':{'country':'Ecuador','alpha2':'EC','alpha3':'ECU','numeric':'218'}, + 'EG':{'country':'Egypt','alpha2':'EG','alpha3':'EGY','numeric':'818'}, + 'SV':{'country':'El Salvador','alpha2':'SV','alpha3':'SLV','numeric':'222'}, + 'GQ':{'country':'Equatorial Guinea','alpha2':'GQ','alpha3':'GNQ','numeric':'226'}, + 'ER':{'country':'Eritrea','alpha2':'ER','alpha3':'ERI','numeric':'232'}, + 'EE':{'country':'Estonia','alpha2':'EE','alpha3':'EST','numeric':'233'}, + 'ET':{'country':'Ethiopia','alpha2':'ET','alpha3':'ETH','numeric':'231'}, + 'FK':{'country':'Falkland Islands','alpha2':'FK','alpha3':'FLK','numeric':'238'}, + 'FO':{'country':'Faroe Islands','alpha2':'FO','alpha3':'FRO','numeric':'234'}, + 'FJ':{'country':'Fiji','alpha2':'FJ','alpha3':'FJI','numeric':'242'}, + 'FI':{'country':'Finland','alpha2':'FI','alpha3':'FIN','numeric':'246'}, + 'FR':{'country':'France','alpha2':'FR','alpha3':'FRA','numeric':'250'}, + 'GF':{'country':'French Guiana','alpha2':'GF','alpha3':'GUF','numeric':'254'}, + 'PF':{'country':'French Polynesia','alpha2':'PF','alpha3':'PYF','numeric':'258'}, + 'TF':{'country':'French Southern Territories','alpha2':'TF','alpha3':'ATF','numeric':'260'}, + 'GA':{'country':'Gabon','alpha2':'GA','alpha3':'GAB','numeric':'266'}, + 'GM':{'country':'Gambia','alpha2':'GM','alpha3':'GMB','numeric':'270'}, + 'GE':{'country':'Georgia','alpha2':'GE','alpha3':'GEO','numeric':'268'}, + 'DE':{'country':'Germany','alpha2':'DE','alpha3':'DEU','numeric':'276'}, + 'GH':{'country':'Ghana','alpha2':'GH','alpha3':'GHA','numeric':'288'}, + 'GI':{'country':'Gibraltar','alpha2':'GI','alpha3':'GIB','numeric':'292'}, + 'GR':{'country':'Greece','alpha2':'GR','alpha3':'GRC','numeric':'300'}, + 'GL':{'country':'Greenland','alpha2':'GL','alpha3':'GRL','numeric':'304'}, + 'GD':{'country':'Grenada','alpha2':'GD','alpha3':'GRD','numeric':'308'}, + 'GP':{'country':'Guadeloupe','alpha2':'GP','alpha3':'GLP','numeric':'312'}, + 'GU':{'country':'Guam','alpha2':'GU','alpha3':'GUM','numeric':'316'}, + 'GT':{'country':'Guatemala','alpha2':'GT','alpha3':'GTM','numeric':'320'}, + 'GG':{'country':'Guernsey','alpha2':'GG','alpha3':'GGY','numeric':'831'}, + 'GN':{'country':'Guinea','alpha2':'GN','alpha3':'GIN','numeric':'324'}, + 'GW':{'country':'Guinea-Bissau','alpha2':'GW','alpha3':'GNB','numeric':'624'}, + 'GY':{'country':'Guyana','alpha2':'GY','alpha3':'GUY','numeric':'328'}, + 'HT':{'country':'Haiti','alpha2':'HT','alpha3':'HTI','numeric':'332'}, + 'HM':{'country':'Heard Island and McDonald Islands','alpha2':'HM','alpha3':'HMD','numeric':'334'}, + 'VA':{'country':'Holy See','alpha2':'VA','alpha3':'VAT','numeric':'336'}, + 'HN':{'country':'Honduras','alpha2':'HN','alpha3':'HND','numeric':'340'}, + 'HK':{'country':'Hong Kong','alpha2':'HK','alpha3':'HKG','numeric':'344'}, + 'HU':{'country':'Hungary','alpha2':'HU','alpha3':'HUN','numeric':'348'}, + 'IS':{'country':'Iceland','alpha2':'IS','alpha3':'ISL','numeric':'352'}, + 'IN':{'country':'India','alpha2':'IN','alpha3':'IND','numeric':'356'}, + 'ID':{'country':'Indonesia','alpha2':'ID','alpha3':'IDN','numeric':'360'}, + 'IR':{'country':'Islamic Republic of Iran','alpha2':'IR','alpha3':'IRN','numeric':'364'}, + 'IQ':{'country':'Iraq','alpha2':'IQ','alpha3':'IRQ','numeric':'368'}, + 'IE':{'country':'Ireland','alpha2':'IE','alpha3':'IRL','numeric':'372'}, + 'IM':{'country':'Isle of Man','alpha2':'IM','alpha3':'IMN','numeric':'833'}, + 'IL':{'country':'Israel','alpha2':'IL','alpha3':'ISR','numeric':'376'}, + 'IT':{'country':'Italy','alpha2':'IT','alpha3':'ITA','numeric':'380'}, + 'JM':{'country':'Jamaica','alpha2':'JM','alpha3':'JAM','numeric':'388'}, + 'JP':{'country':'Japan','alpha2':'JP','alpha3':'JPN','numeric':'392'}, + 'JE':{'country':'Jersey','alpha2':'JE','alpha3':'JEY','numeric':'832'}, + 'JO':{'country':'Jordan','alpha2':'JO','alpha3':'JOR','numeric':'400'}, + 'KZ':{'country':'Kazakhstan','alpha2':'KZ','alpha3':'KAZ','numeric':'398'}, + 'KE':{'country':'Kenya','alpha2':'KE','alpha3':'KEN','numeric':'404'}, + 'KI':{'country':'Kiribati','alpha2':'KI','alpha3':'KIR','numeric':'296'}, + 'KP':{'country':'Democratic People\'s Republic of Korea','alpha2':'KP','alpha3':'PRK','numeric':'408'}, + 'KR':{'country':'Republic of Korea','alpha2':'KR','alpha3':'KOR','numeric':'410'}, + 'KW':{'country':'Kuwait','alpha2':'KW','alpha3':'KWT','numeric':'414'}, + 'KG':{'country':'Kyrgyzstan','alpha2':'KG','alpha3':'KGZ','numeric':'417'}, + 'LA':{'country':'Lao People\'s Democratic Republic','alpha2':'LA','alpha3':'LAO','numeric':'418'}, + 'LV':{'country':'Latvia','alpha2':'LV','alpha3':'LVA','numeric':'428'}, + 'LB':{'country':'Lebanon','alpha2':'LB','alpha3':'LBN','numeric':'422'}, + 'LS':{'country':'Lesotho','alpha2':'LS','alpha3':'LSO','numeric':'426'}, + 'LR':{'country':'Liberia','alpha2':'LR','alpha3':'LBR','numeric':'430'}, + 'LY':{'country':'Libya','alpha2':'LY','alpha3':'LBY','numeric':'434'}, + 'LI':{'country':'Liechtenstein','alpha2':'LI','alpha3':'LIE','numeric':'438'}, + 'LT':{'country':'Lithuania','alpha2':'LT','alpha3':'LTU','numeric':'440'}, + 'LU':{'country':'Luxembourg','alpha2':'LU','alpha3':'LUX','numeric':'442'}, + 'MO':{'country':'Macao','alpha2':'MO','alpha3':'MAC','numeric':'446'}, + 'MK':{'country':'Macedonia','alpha2':'MK','alpha3':'MKD','numeric':'807'}, + 'MG':{'country':'Madagascar','alpha2':'MG','alpha3':'MDG','numeric':'450'}, + 'MW':{'country':'Malawi','alpha2':'MW','alpha3':'MWI','numeric':'454'}, + 'MY':{'country':'Malaysia','alpha2':'MY','alpha3':'MYS','numeric':'458'}, + 'MV':{'country':'Maldives','alpha2':'MV','alpha3':'MDV','numeric':'462'}, + 'ML':{'country':'Mali','alpha2':'ML','alpha3':'MLI','numeric':'466'}, + 'MT':{'country':'Malta','alpha2':'MT','alpha3':'MLT','numeric':'470'}, + 'MH':{'country':'Marshall Islands','alpha2':'MH','alpha3':'MHL','numeric':'584'}, + 'MQ':{'country':'Martinique','alpha2':'MQ','alpha3':'MTQ','numeric':'474'}, + 'MR':{'country':'Mauritania','alpha2':'MR','alpha3':'MRT','numeric':'478'}, + 'MU':{'country':'Mauritius','alpha2':'MU','alpha3':'MUS','numeric':'480'}, + 'YT':{'country':'Mayotte','alpha2':'YT','alpha3':'MYT','numeric':'175'}, + 'MX':{'country':'Mexico','alpha2':'MX','alpha3':'MEX','numeric':'484'}, + 'FM':{'country':'Federated States of Micronesia','alpha2':'FM','alpha3':'FSM','numeric':'583'}, + 'MD':{'country':'Republic of Moldova','alpha2':'MD','alpha3':'MDA','numeric':'498'}, + 'MC':{'country':'Monaco','alpha2':'MC','alpha3':'MCO','numeric':'492'}, + 'MN':{'country':'Mongolia','alpha2':'MN','alpha3':'MNG','numeric':'496'}, + 'ME':{'country':'Montenegro','alpha2':'ME','alpha3':'MNE','numeric':'499'}, + 'MS':{'country':'Montserrat','alpha2':'MS','alpha3':'MSR','numeric':'500'}, + 'MA':{'country':'Morocco','alpha2':'MA','alpha3':'MAR','numeric':'504'}, + 'MZ':{'country':'Mozambique','alpha2':'MZ','alpha3':'MOZ','numeric':'508'}, + 'MM':{'country':'Myanmar','alpha2':'MM','alpha3':'MMR','numeric':'104'}, + 'NA':{'country':'Namibia','alpha2':'NA','alpha3':'NAM','numeric':'516'}, + 'NR':{'country':'Nauru','alpha2':'NR','alpha3':'NRU','numeric':'520'}, + 'NP':{'country':'Nepal','alpha2':'NP','alpha3':'NPL','numeric':'524'}, + 'NL':{'country':'Netherlands','alpha2':'NL','alpha3':'NLD','numeric':'528'}, + 'NC':{'country':'New Caledonia','alpha2':'NC','alpha3':'NCL','numeric':'540'}, + 'NZ':{'country':'New Zealand','alpha2':'NZ','alpha3':'NZL','numeric':'554'}, + 'NI':{'country':'Nicaragua','alpha2':'NI','alpha3':'NIC','numeric':'558'}, + 'NE':{'country':'Niger','alpha2':'NE','alpha3':'NER','numeric':'562'}, + 'NG':{'country':'Nigeria','alpha2':'NG','alpha3':'NGA','numeric':'566'}, + 'NU':{'country':'Niue','alpha2':'NU','alpha3':'NIU','numeric':'570'}, + 'NF':{'country':'Norfolk Island','alpha2':'NF','alpha3':'NFK','numeric':'574'}, + 'MP':{'country':'Northern Mariana Islands','alpha2':'MP','alpha3':'MNP','numeric':'580'}, + 'NO':{'country':'Norway','alpha2':'NO','alpha3':'NOR','numeric':'578'}, + 'OM':{'country':'Oman','alpha2':'OM','alpha3':'OMN','numeric':'512'}, + 'PK':{'country':'Pakistan','alpha2':'PK','alpha3':'PAK','numeric':'586'}, + 'PW':{'country':'Palau','alpha2':'PW','alpha3':'PLW','numeric':'585'}, + 'PS':{'country':'State of Palestine','alpha2':'PS','alpha3':'PSE','numeric':'275'}, + 'PA':{'country':'Panama','alpha2':'PA','alpha3':'PAN','numeric':'591'}, + 'PG':{'country':'Papua New Guinea','alpha2':'PG','alpha3':'PNG','numeric':'598'}, + 'PY':{'country':'Paraguay','alpha2':'PY','alpha3':'PRY','numeric':'600'}, + 'PE':{'country':'Peru','alpha2':'PE','alpha3':'PER','numeric':'604'}, + 'PH':{'country':'Philippines','alpha2':'PH','alpha3':'PHL','numeric':'608'}, + 'PN':{'country':'Pitcairn','alpha2':'PN','alpha3':'PCN','numeric':'612'}, + 'PL':{'country':'Poland','alpha2':'PL','alpha3':'POL','numeric':'616'}, + 'PT':{'country':'Portugal','alpha2':'PT','alpha3':'PRT','numeric':'620'}, + 'PR':{'country':'Puerto Rico','alpha2':'PR','alpha3':'PRI','numeric':'630'}, + 'QA':{'country':'Qatar','alpha2':'QA','alpha3':'QAT','numeric':'634'}, + 'RE':{'country':'Réunion','alpha2':'RE','alpha3':'REU','numeric':'638'}, + 'RO':{'country':'Romania','alpha2':'RO','alpha3':'ROU','numeric':'642'}, + 'RU':{'country':'Russian Federation','alpha2':'RU','alpha3':'RUS','numeric':'643'}, + 'RW':{'country':'Rwanda','alpha2':'RW','alpha3':'RWA','numeric':'646'}, + 'BL':{'country':'Saint Barthélemy','alpha2':'BL','alpha3':'BLM','numeric':'652'}, + 'SH':{'country':'Saint Helena, Ascension and Tristan da Cunha','alpha2':'SH','alpha3':'SHN','numeric':'654'}, + 'KN':{'country':'Saint Kitts and Nevis','alpha2':'KN','alpha3':'KNA','numeric':'659'}, + 'LC':{'country':'Saint Lucia','alpha2':'LC','alpha3':'LCA','numeric':'662'}, + 'MF':{'country':'Saint Martin','alpha2':'MF','alpha3':'MAF','numeric':'663'}, + 'PM':{'country':'Saint Pierre and Miquelon','alpha2':'PM','alpha3':'SPM','numeric':'666'}, + 'VC':{'country':'Saint Vincent and the Grenadines','alpha2':'VC','alpha3':'VCT','numeric':'670'}, + 'WS':{'country':'Samoa','alpha2':'WS','alpha3':'WSM','numeric':'882'}, + 'SM':{'country':'San Marino','alpha2':'SM','alpha3':'SMR','numeric':'674'}, + 'ST':{'country':'Sao Tome and Principe','alpha2':'ST','alpha3':'STP','numeric':'678'}, + 'SA':{'country':'Saudi Arabia','alpha2':'SA','alpha3':'SAU','numeric':'682'}, + 'SN':{'country':'Senegal','alpha2':'SN','alpha3':'SEN','numeric':'686'}, + 'RS':{'country':'Serbia','alpha2':'RS','alpha3':'SRB','numeric':'688'}, + 'SC':{'country':'Seychelles','alpha2':'SC','alpha3':'SYC','numeric':'690'}, + 'SL':{'country':'Sierra Leone','alpha2':'SL','alpha3':'SLE','numeric':'694'}, + 'SG':{'country':'Singapore','alpha2':'SG','alpha3':'SGP','numeric':'702'}, + 'SX':{'country':'Sint Maarten','alpha2':'SX','alpha3':'SXM','numeric':'534'}, + 'SK':{'country':'Slovakia','alpha2':'SK','alpha3':'SVK','numeric':'703'}, + 'SI':{'country':'Slovenia','alpha2':'SI','alpha3':'SVN','numeric':'705'}, + 'SB':{'country':'Solomon Islands','alpha2':'SB','alpha3':'SLB','numeric':'090'}, + 'SO':{'country':'Somalia','alpha2':'SO','alpha3':'SOM','numeric':'706'}, + 'ZA':{'country':'South Africa','alpha2':'ZA','alpha3':'ZAF','numeric':'710'}, + 'GS':{'country':'South Georgia and the South Sandwich Islands','alpha2':'GS','alpha3':'SGS','numeric':'239'}, + 'SS':{'country':'South Sudan','alpha2':'SS','alpha3':'SSD','numeric':'728'}, + 'ES':{'country':'Spain','alpha2':'ES','alpha3':'ESP','numeric':'724'}, + 'LK':{'country':'Sri Lanka','alpha2':'LK','alpha3':'LKA','numeric':'144'}, + 'SD':{'country':'Sudan','alpha2':'SD','alpha3':'SDN','numeric':'729'}, + 'SR':{'country':'Suriname','alpha2':'SR','alpha3':'SUR','numeric':'740'}, + 'SJ':{'country':'Svalbard and Jan Mayen','alpha2':'SJ','alpha3':'SJM','numeric':'744'}, + 'SZ':{'country':'Swaziland','alpha2':'SZ','alpha3':'SWZ','numeric':'748'}, + 'SE':{'country':'Sweden','alpha2':'SE','alpha3':'SWE','numeric':'752'}, + 'CH':{'country':'Switzerland','alpha2':'CH','alpha3':'CHE','numeric':'756'}, + 'SY':{'country':'Syrian Arab Republic','alpha2':'SY','alpha3':'SYR','numeric':'760'}, + 'TW':{'country':'Taiwan, Province of China','alpha2':'TW','alpha3':'TWN','numeric':'158'}, + 'TJ':{'country':'Tajikistan','alpha2':'TJ','alpha3':'TJK','numeric':'762'}, + 'TZ':{'country':'United Republic of Tanzania','alpha2':'TZ','alpha3':'TZA','numeric':'834'}, + 'TH':{'country':'Thailand','alpha2':'TH','alpha3':'THA','numeric':'764'}, + 'TL':{'country':'Timor-Leste','alpha2':'TL','alpha3':'TLS','numeric':'626'}, + 'TG':{'country':'Togo','alpha2':'TG','alpha3':'TGO','numeric':'768'}, + 'TK':{'country':'Tokelau','alpha2':'TK','alpha3':'TKL','numeric':'772'}, + 'TO':{'country':'Tonga','alpha2':'TO','alpha3':'TON','numeric':'776'}, + 'TT':{'country':'Trinidad and Tobago','alpha2':'TT','alpha3':'TTO','numeric':'780'}, + 'TN':{'country':'Tunisia','alpha2':'TN','alpha3':'TUN','numeric':'788'}, + 'TR':{'country':'Turkey','alpha2':'TR','alpha3':'TUR','numeric':'792'}, + 'TM':{'country':'Turkmenistan','alpha2':'TM','alpha3':'TKM','numeric':'795'}, + 'TC':{'country':'Turks and Caicos Islands','alpha2':'TC','alpha3':'TCA','numeric':'796'}, + 'TV':{'country':'Tuvalu','alpha2':'TV','alpha3':'TUV','numeric':'798'}, + 'UG':{'country':'Uganda','alpha2':'UG','alpha3':'UGA','numeric':'800'}, + 'UA':{'country':'Ukraine','alpha2':'UA','alpha3':'UKR','numeric':'804'}, + 'AE':{'country':'United Arab Emirates','alpha2':'AE','alpha3':'ARE','numeric':'784'}, + 'GB':{'country':'United Kingdom of Great Britain and Northern Ireland','alpha2':'GB','alpha3':'GBR','numeric':'826'}, + 'US':{'country':'United States of America','alpha2':'US','alpha3':'USA','numeric':'840'}, + 'UM':{'country':'United States Minor Outlying Islands','alpha2':'UM','alpha3':'UMI','numeric':'581'}, + 'UY':{'country':'Uruguay','alpha2':'UY','alpha3':'URY','numeric':'858'}, + 'UZ':{'country':'Uzbekistan','alpha2':'UZ','alpha3':'UZB','numeric':'860'}, + 'VU':{'country':'Vanuatu','alpha2':'VU','alpha3':'VUT','numeric':'548'}, + 'VE':{'country':'Venezuela (Bolivarian Republic of)','alpha2':'VE','alpha3':'VEN','numeric':'862'}, + 'VN':{'country':'Viet Nam','alpha2':'VN','alpha3':'VNM','numeric':'704'}, + 'VG':{'country':'Virgin Islands','alpha2':'VG','alpha3':'VGB','numeric':'092'}, + 'VI':{'country':'Virgin Islands','alpha2':'VI','alpha3':'VIR','numeric':'850'}, + 'WF':{'country':'Wallis and Futuna','alpha2':'WF','alpha3':'WLF','numeric':'876'}, + 'EH':{'country':'Western Sahara','alpha2':'EH','alpha3':'ESH','numeric':'732'}, + 'YE':{'country':'Yemen','alpha2':'YE','alpha3':'YEM','numeric':'887'}, + 'ZM':{'country':'Zambia','alpha2':'ZM','alpha3':'ZMB','numeric':'894'}, + 'ZW':{'country':'Zimbabwe','alpha2':'ZW','alpha3':'ZWE','numeric':'716'}}; + +const iso3to2 = {'AFG':'AF','ALA':'AX','ALB':'AL','DZA':'DZ','ASM':'AS','AND':'AD','AGO':'AO', + 'AIA':'AI','ATA':'AQ','ATG':'AG','ARG':'AR','ARM':'AM','ABW':'AW','AUS':'AU', + 'AUT':'AT','AZE':'AZ','BHS':'BS','BHR':'BH','BGD':'BD','BRB':'BB','BLR':'BY', + 'BEL':'BE','BLZ':'BZ','BEN':'BJ','BMU':'BM','BTN':'BT','BOL':'BO','BES':'BQ', + 'BIH':'BA','BWA':'BW','BVT':'BV','BRA':'BR','IOT':'IO','BRN':'BN','BGR':'BG', + 'BFA':'BF','BDI':'BI','CPV':'CV','KHM':'KH','CMR':'CM','CAN':'CA','CYM':'KY', + 'CAF':'CF','TCD':'TD','CHL':'CL','CHN':'CN','CXR':'CX','CCK':'CC','COL':'CO', + 'COM':'KM','COG':'CG','COD':'CD','COK':'CK','CRI':'CR','CIV':'CI','HRV':'HR', + 'CUB':'CU','CUW':'CW','CYP':'CY','CZE':'CZ','DNK':'DK','DJI':'DJ','DMA':'DM', + 'DOM':'DO','ECU':'EC','EGY':'EG','SLV':'SV','GNQ':'GQ','ERI':'ER','EST':'EE', + 'ETH':'ET','FLK':'FK','FRO':'FO','FJI':'FJ','FIN':'FI','FRA':'FR','GUF':'GF', + 'PYF':'PF','ATF':'TF','GAB':'GA','GMB':'GM','GEO':'GE','DEU':'DE','GHA':'GH', + 'GIB':'GI','GRC':'GR','GRL':'GL','GRD':'GD','GLP':'GP','GUM':'GU','GTM':'GT', + 'GGY':'GG','GIN':'GN','GNB':'GW','GUY':'GY','HTI':'HT','HMD':'HM','VAT':'VA', + 'HND':'HN','HKG':'HK','HUN':'HU','ISL':'IS','IND':'IN','IDN':'ID','IRN':'IR', + 'IRQ':'IQ','IRL':'IE','IMN':'IM','ISR':'IL','ITA':'IT','JAM':'JM','JPN':'JP', + 'JEY':'JE','JOR':'JO','KAZ':'KZ','KEN':'KE','KIR':'KI','PRK':'KP','KOR':'KR', + 'KWT':'KW','KGZ':'KG','LAO':'LA','LVA':'LV','LBN':'LB','LSO':'LS','LBR':'LR', + 'LBY':'LY','LIE':'LI','LTU':'LT','LUX':'LU','MAC':'MO','MKD':'MK','MDG':'MG', + 'MWI':'MW','MYS':'MY','MDV':'MV','MLI':'ML','MLT':'MT','MHL':'MH','MTQ':'MQ', + 'MRT':'MR','MUS':'MU','MYT':'YT','MEX':'MX','FSM':'FM','MDA':'MD','MCO':'MC', + 'MNG':'MN','MNE':'ME','MSR':'MS','MAR':'MA','MOZ':'MZ','MMR':'MM','NAM':'NA', + 'NRU':'NR','NPL':'NP','NLD':'NL','NCL':'NC','NZL':'NZ','NIC':'NI','NER':'NE', + 'NGA':'NG','NIU':'NU','NFK':'NF','MNP':'MP','NOR':'NO','OMN':'OM','PAK':'PK', + 'PLW':'PW','PSE':'PS','PAN':'PA','PNG':'PG','PRY':'PY','PER':'PE','PHL':'PH', + 'PCN':'PN','POL':'PL','PRT':'PT','PRI':'PR','QAT':'QA','REU':'RE','ROU':'RO', + 'RUS':'RU','RWA':'RW','BLM':'BL','SHN':'SH','KNA':'KN','LCA':'LC','MAF':'MF', + 'SPM':'PM','VCT':'VC','WSM':'WS','SMR':'SM','STP':'ST','SAU':'SA','SEN':'SN', + 'SRB':'RS','SYC':'SC','SLE':'SL','SGP':'SG','SXM':'SX','SVK':'SK','SVN':'SI', + 'SLB':'SB','SOM':'SO','ZAF':'ZA','SGS':'GS','SSD':'SS','ESP':'ES','LKA':'LK', + 'SDN':'SD','SUR':'SR','SJM':'SJ','SWZ':'SZ','SWE':'SE','CHE':'CH','SYR':'SY', + 'TWN':'TW','TJK':'TJ','TZA':'TZ','THA':'TH','TLS':'TL','TGO':'TG','TKL':'TK', + 'TON':'TO','TTO':'TT','TUN':'TN','TUR':'TR','TKM':'TM','TCA':'TC','TUV':'TV', + 'UGA':'UG','UKR':'UA','ARE':'AE','GBR':'GB','USA':'US','UMI':'UM','URY':'UY', + 'UZB':'UZ','VUT':'VU','VEN':'VE','VNM':'VN','VGB':'VG','VIR':'VI','WLF':'WF', + 'ESH':'EH','YEM':'YE','ZMB':'ZM','ZWE':'ZW'}; + + +function isISO2Code(code) { + return byISO2[_.toUpper(code)] !== undefined; +} + +function isISO3Code(code) { + return iso3to2[_.toUpper(code)] !== undefined; +} + +function convertISO2ToISO3(code) { + const country = byISO2[_.toUpper(code)]; + if (country !== undefined) { + return country.alpha3; + } +} + +function convertISO3ToISO2(code) { + return iso3to2[_.toUpper(code)]; +} + +function iso3Code(code) { + if (isISO2Code(code)) { + return convertISO2ToISO3(code); + } else if (isISO3Code(code)) { + return _.toUpper(code); + } +} + +module.exports = { + isISO2Code, + isISO3Code, + convertISO2ToISO3, + convertISO3ToISO2, + iso3Code +}; diff --git a/package.json b/package.json index 9c6e977e..2d1f2f06 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,6 @@ "geojson": "^0.5.0", "geolib": "^2.0.18", "iso-639-3": "^1.0.0", - "iso3166-1": "^0.3.0", "joi": "^13.1.3", "locale": "^0.1.0", "lodash": "^4.17.4", diff --git a/sanitizer/_boundary_country.js b/sanitizer/_boundary_country.js index 2b7f9bb3..81a5e7d5 100644 --- a/sanitizer/_boundary_country.js +++ b/sanitizer/_boundary_country.js @@ -1,5 +1,5 @@ const check = require('check-types'); -const iso3166 = require('iso3166-1'); +const iso3166 = require('../helper/iso3166'); function _sanitize(raw, clean) { // error & warning messages @@ -18,7 +18,7 @@ function _sanitize(raw, clean) { } // must be a valid ISO 3166 code - else if (!containsIsoCode(country.toUpperCase())) { + else if (!containsIsoCode(country)) { messages.errors.push(country + ' is not a valid ISO2/ISO3 country code'); } @@ -26,7 +26,7 @@ function _sanitize(raw, clean) { else { // the only way for boundary.country to be assigned is if input is // a string and a known ISO2 or ISO3 - clean['boundary.country'] = iso3166.to3(country.toUpperCase()); + clean['boundary.country'] = iso3166.iso3Code(country); } } @@ -34,7 +34,7 @@ function _sanitize(raw, clean) { } function containsIsoCode(isoCode) { - return iso3166.is2(isoCode) || iso3166.is3(isoCode); + return iso3166.isISO2Code(isoCode) || iso3166.isISO3Code(isoCode); } function _expected(){ diff --git a/sanitizer/_iso2_to_iso3.js b/sanitizer/_iso2_to_iso3.js index 6b37cec2..882f80d7 100644 --- a/sanitizer/_iso2_to_iso3.js +++ b/sanitizer/_iso2_to_iso3.js @@ -1,5 +1,5 @@ const _ = require('lodash'); -const iso3166 = require('iso3166-1'); +const iso3166 = require('../helper/iso3166'); // this sanitizer exists solely to convert an ISO2 country value to ISO3 // eg - 'TH' -> 'THA' @@ -8,8 +8,8 @@ function _sanitize( raw, clean ){ // error & warning messages const messages = { errors: [], warnings: [] }; - if (clean.hasOwnProperty('parsed_text') && iso3166.is2(_.toUpper(clean.parsed_text.country))) { - clean.parsed_text.country = iso3166.to3(_.toUpper(clean.parsed_text.country)); + if (clean.hasOwnProperty('parsed_text') && iso3166.isISO2Code(clean.parsed_text.country)) { + clean.parsed_text.country = iso3166.convertISO2ToISO3(clean.parsed_text.country); } return messages; diff --git a/test/unit/helper/iso3166.js b/test/unit/helper/iso3166.js new file mode 100644 index 00000000..0d78715b --- /dev/null +++ b/test/unit/helper/iso3166.js @@ -0,0 +1,67 @@ +var iso3166 = require('../../../helper/iso3166'); + +module.exports.tests = {}; + +module.exports.tests.recognizingISOCodes = function(test, common) { + test('Recognizes iso2 codes', function(t) { + t.true(iso3166.isISO2Code('US')); + t.true(iso3166.isISO2Code('Us')); + t.false(iso3166.isISO2Code('xx')); + t.end(); + }); + + test('Recognizes iso3 codes', function(t) { + t.true(iso3166.isISO3Code('USA')); + t.true(iso3166.isISO3Code('UsA')); + t.false(iso3166.isISO3Code('xxx')); + t.end(); + }); +}; + +module.exports.tests.convertingISOCodes = function(test, common) { + test('converts iso2 to iso3', function(t) { + t.equal('USA', iso3166.convertISO2ToISO3('uS')); + t.equal('FRA', iso3166.convertISO2ToISO3('FR')); + t.equal('FRA', iso3166.convertISO2ToISO3('Fr')); + t.equal(undefined, iso3166.convertISO2ToISO3('uSa')); + t.equal(undefined, iso3166.convertISO2ToISO3('xx')); + t.end(); + }); + + test('converts iso3 to iso2', function(t) { + t.equal('US', iso3166.convertISO3ToISO2('uSa')); + t.equal('FR', iso3166.convertISO3ToISO2('fra')); + t.equal('FR', iso3166.convertISO3ToISO2('frA')); + t.equal(undefined, iso3166.convertISO3ToISO2('xxx')); + t.equal(undefined, iso3166.convertISO3ToISO2('fr')); + t.end(); + }); +}; + +module.exports.tests.getISO3Code = function(test, common) { + test('Gets iso 3 code for iso 2 code', function(t) { + t.equal('USA', iso3166.iso3Code('uS')); + t.equal('FRA', iso3166.iso3Code('fr')); + t.equal(undefined, iso3166.iso3Code('xxx')); + t.end(); + }); + + test('Recognizes and returns existing ISO 3 code', function(t) { + t.equal('USA', iso3166.iso3Code('USA')); + t.equal('FRA', iso3166.iso3Code('FRA')); + t.end(); + }); + + test('Upcases given ISO3 code if needed', function(t) { + t.equal('USA', iso3166.iso3Code('UsA')); + t.equal('USA', iso3166.iso3Code('usa')); + t.equal('FRA', iso3166.iso3Code('FRa')); + t.end(); + }); +}; + +module.exports.all = function (test, common) { + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/unit/run.js b/test/unit/run.js index f196dc53..c8eee73a 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -37,6 +37,7 @@ var tests = [ require('./helper/fieldValue'), require('./helper/geojsonify_place_details'), require('./helper/geojsonify'), + require('./helper/iso3166'), require('./helper/logging'), require('./helper/TypeMapping'), require('./helper/type_mapping'), From 800eb8ca0363b3d77793a343ccade9e81ccc37ca Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sun, 15 Jul 2018 12:17:41 -0400 Subject: [PATCH 02/17] Move lots of logging from info to debug These are log lines that are not really useful in a production context, and just create a lot of noise. --- controller/placeholder.js | 2 +- middleware/dedupe.js | 4 ++-- middleware/requestLanguage.js | 3 +-- query/search.js | 4 ++-- test/unit/controller/placeholder.js | 26 +++++++++++++------------- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/controller/placeholder.js b/controller/placeholder.js index edba469a..84637116 100644 --- a/controller/placeholder.js +++ b/controller/placeholder.js @@ -265,7 +265,7 @@ function setup(placeholderService, do_geometric_filters_apply, should_execute) { `[result_count:${_.defaultTo(res.data, []).length}]` ]; - logger.info(messageParts.join(' ')); + logger.debug(messageParts.join(' ')); debugLog.push(req, messageParts[1].slice(1,-1)); debugLog.push(req, res.data); } diff --git a/middleware/dedupe.js b/middleware/dedupe.js index 0a3fd7e6..4b637ed5 100644 --- a/middleware/dedupe.js +++ b/middleware/dedupe.js @@ -36,7 +36,7 @@ function dedupeResults(req, res, next) { // since the order in which Elasticsearch returns identical text matches is arbitrary // of course, if the new one is preferred we should replace previous with new else if (isPreferred(uniqueResults[dupeIndex], hit)) { - logger.info('[dupe][replacing]', { + logger.debug('[dupe][replacing]', { query: req.clean.text, previous: uniqueResults[dupeIndex].source, hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id @@ -46,7 +46,7 @@ function dedupeResults(req, res, next) { } // if not preferred over existing, just log and move on else { - logger.info('[dupe][skipping]', { + logger.debug('[dupe][skipping]', { query: req.clean.text, previous: uniqueResults[dupeIndex].source, hit: field.getStringValue(hit.name.default) + ' ' + hit.source + ':' + hit._id diff --git a/middleware/requestLanguage.js b/middleware/requestLanguage.js index d6e154ab..c15cd671 100644 --- a/middleware/requestLanguage.js +++ b/middleware/requestLanguage.js @@ -101,8 +101,7 @@ module.exports = function middleware( req, res, next ){ defaulted: req.language.defaulted }; - // logging - logger.info( '[lang] \'%s\' via \'%s\'', lang.iso6391, via ); + logger.debug( '[lang] \'%s\' via \'%s\'', lang.iso6391, via ); next(); }; diff --git a/query/search.js b/query/search.js index 26ae1e40..43990a33 100644 --- a/query/search.js +++ b/query/search.js @@ -120,10 +120,10 @@ function generateQuery( clean ){ //console.log(JSON.stringify(q, null, 2)); if (q !== undefined) { - logger.info(logStr); + logger.debug(logStr); } else { - logger.info('[parser:libpostal] query type not supported'); + logger.debug('[parser:libpostal] query type not supported'); } return q; diff --git a/test/unit/controller/placeholder.js b/test/unit/controller/placeholder.js index 82ace9bd..6d9df491 100644 --- a/test/unit/controller/placeholder.js +++ b/test/unit/controller/placeholder.js @@ -291,7 +291,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:2]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:2]')); t.end(); }); @@ -355,7 +355,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:1]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:1]')); t.end(); }); @@ -415,7 +415,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:1]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:1]')); t.end(); }); @@ -473,7 +473,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:1]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:1]')); t.end(); }); @@ -538,7 +538,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:1]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:1]')); }); }); @@ -607,7 +607,7 @@ module.exports.tests.success = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:1]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:1]')); t.end(); }); @@ -1403,7 +1403,7 @@ module.exports.tests.result_filtering = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:3]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:3]')); t.end(); }); @@ -1538,7 +1538,7 @@ module.exports.tests.result_filtering = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:3]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:3]')); t.end(); }); @@ -1680,7 +1680,7 @@ module.exports.tests.result_filtering = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:2]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:2]')); t.end(); }); @@ -1852,7 +1852,7 @@ module.exports.tests.result_filtering = (test, common) => { }; t.deepEquals(res, expected_res); - t.ok(logger.isInfoMessage('[controller:placeholder] [result_count:3]')); + t.ok(logger.isDebugMessage('[controller:placeholder] [result_count:3]')); t.end(); }); @@ -2356,7 +2356,7 @@ module.exports.tests.error_conditions = (test, common) => { controller(req, res, () => { t.deepEquals(res, {}, 'res should not have been modified'); t.deepEquals(req.errors, ['placeholder service error']); - t.notOk(logger.isInfoMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); + t.notOk(logger.isDebugMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); t.end(); }); @@ -2385,7 +2385,7 @@ module.exports.tests.error_conditions = (test, common) => { controller(req, res, () => { t.deepEquals(res, {}, 'res should not have been modified'); t.deepEquals(req.errors, ['placeholder service error']); - t.notOk(logger.isInfoMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); + t.notOk(logger.isDebugMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); t.end(); }); @@ -2410,7 +2410,7 @@ module.exports.tests.error_conditions = (test, common) => { controller(req, res, () => { t.deepEquals(res, {}, 'res should not have been modified'); t.deepEquals(req.errors, [{ error_key: 'error_value' }]); - t.notOk(logger.isInfoMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); + t.notOk(logger.isDebugMessage(/\\[controller:placeholder\\] \\[result_count:\\d+\\]/)); t.end(); }); From 67a3ab72a9015c3a172fa846a5565011a3c32b69 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Sat, 1 Sep 2018 16:51:17 +0000 Subject: [PATCH 03/17] fix(package): update pelias-logger to version 1.2.0 Closes #1189 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2d1f2f06..6f2ecf8d 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "pelias-categories": "^1.2.0", "pelias-config": "^3.0.2", "pelias-labels": "^1.8.0", - "pelias-logger": "^0.4.2", + "pelias-logger": "^1.2.0", "pelias-microservice-wrapper": "^1.4.0", "pelias-model": "^5.5.2", "pelias-query": "^9.1.1", From 7d9e3e29fd49ebdd3aa24d6e6484d7889d352b06 Mon Sep 17 00:00:00 2001 From: Joxit Date: Sun, 26 Aug 2018 23:31:44 +0200 Subject: [PATCH 04/17] feat(findbyid): Add lang query param for placeholder This will reduce network transfer and speedup requests Related: https://github.com/pelias/placeholder/pull/128 --- service/configurations/Language.js | 10 ++++++++-- test/unit/service/configurations/Language.js | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/service/configurations/Language.js b/service/configurations/Language.js index c20fa861..a4d96c96 100644 --- a/service/configurations/Language.js +++ b/service/configurations/Language.js @@ -15,12 +15,18 @@ class Language extends ServiceConfiguration { Array.prototype.push.apply(acc, _.values(_.pickBy(doc.parent, (v, k) => _.endsWith(k, '_id') ) ) ); return acc; }, []); - - return { + const lang = _.get(req, 'clean.lang.iso6393'); + const parameters = { // arrays will be nested, so flatten first, then uniqify, and finally join elements with comma ids: _.uniq(_.flattenDeep(ids)).join(',') }; + if (lang) { + parameters.lang = lang; + } + + return parameters; + } getUrl(req) { diff --git a/test/unit/service/configurations/Language.js b/test/unit/service/configurations/Language.js index 72cfb7d3..682ddc0a 100644 --- a/test/unit/service/configurations/Language.js +++ b/test/unit/service/configurations/Language.js @@ -117,6 +117,24 @@ module.exports.tests.all = (test, common) => { }); + + test('getParameters should return lang when req.clean.lang.iso6393 is defined', (t) => { + const configBlob = { + url: 'http://localhost:1234', + timeout: 17, + retries: 19 + }; + + const req = {clean: {lang: {iso6393: 'eng' }}}; + const res = { }; + + const language = new Language(configBlob); + + t.deepEquals(language.getParameters(req, res), { ids: '', lang: 'eng' }); + t.end(); + + }); + test('getHeaders should return empty object', (t) => { const configBlob = { url: 'base url', From 34e3e5fd86f09652e36de0957876ea00ef8a5a7a Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 29 Aug 2018 03:25:53 +0300 Subject: [PATCH 05/17] feat(release): replace semantic-release dep with Travis build stages This change moves semantic-release out of dev-dependencies, but keeps its functionality by calling semantic-release as usual in a `release` TravisCI build stage. There are several advantages to this method: 1.) semantic-release is run only after all builds succeed. Our previous approach could have theoretically run semantic-release when some Node.js versions failed with the current code 2.) semantic-release (and it's many dependencies) are removed from `node_modules`. This increases the speed of `npm install` in all cases, and reduces the size of our Docker images by 20MB (from 284MB to 264MB)! Since the only time semantic-release is needed is on TravisCI anyway, it seems pointless that every installation of Pelias should include it. 3.) Because semantic-release is not in `package.json`, Greenkeeper will not attempt to update it. Semantic release updates _very_ frequently, and each update attempt seems to have a decent chance of experiencing a random TravisCI failure, causing unwanted notifications. There are probably downsides to this approach. For example, we should consider pinning the major version of semantic release during install. Additionally, and for obvious reasons, we can't fully test this change until it's merged to the `production` branch. We should consider testing it first on a lower priority repository. If this change _does_ work well, we should consider adopting it everywhere. --- .travis.yml | 7 +++++-- package.json | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5aec4536..e83588fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,11 @@ matrix: script: npm run travis before_install: - npm i -g npm -after_success: - - npx semantic-release branches: except: - /^v\d+\.\d+\.\d+$/ +jobs: + include: + - stage: release + node_js: 10 + script: npx semantic-release diff --git a/package.json b/package.json index 6f2ecf8d..80521db8 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,6 @@ "pelias-mock-logger": "^1.3.0", "precommit-hook": "^3.0.0", "proxyquire": "^2.0.0", - "semantic-release": "^15.1.4", "source-map": "^0.7.0", "tap-dot": "^2.0.0", "tape": "^4.5.1", From 2de274954f122b042488b385053f8a37e72e9b89 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 30 Aug 2018 23:08:43 +0300 Subject: [PATCH 06/17] Use ci-tools script for semantic-release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e83588fb..5ab053b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,4 @@ jobs: include: - stage: release node_js: 10 - script: npx semantic-release + script: curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash - From b5ed14011158937a140fbd40fc3fcd6403ab5298 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 30 Aug 2018 23:09:51 +0300 Subject: [PATCH 07/17] Use docker image build script from ci-tools repo --- .circleci/config.yml | 2 +- .circleci/docker.sh | 32 -------------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 .circleci/docker.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aeea9df..d8fa41df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,4 +9,4 @@ jobs: - setup_remote_docker - run: name: Build and push image to Docker Hub - command: sh .circleci/docker.sh + command: apk --no-cache add curl && curl "https://raw.githubusercontent.com/pelias/ci-tools/master/build-docker-images.sh" | sh - diff --git a/.circleci/docker.sh b/.circleci/docker.sh deleted file mode 100644 index 0dfa433c..00000000 --- a/.circleci/docker.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -set -u - -# collect params from ENV vars -DATE=`date +%Y-%m-%d` -DOCKER_REPOSITORY="pelias" -DOCKER_PROJECT="${DOCKER_REPOSITORY}/${CIRCLE_PROJECT_REPONAME}" - -BRANCH="$(echo $CIRCLE_BRANCH | tr '/' '-')" #slashes are not valid in docker tags. replace with dashes - -# the name of the image that represents the "branch", that is an image that will be updated over time with the git branch -# the production branch is changed to "latest", otherwise the git branch becomes the name of the version -if [[ "${BRANCH}" == "production" ]]; then - DOCKER_BRANCH_IMAGE_VERSION="latest" -else - DOCKER_BRANCH_IMAGE_VERSION="$BRANCH" -fi -DOCKER_BRANCH_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_BRANCH_IMAGE_VERSION}" - -# the name of the image that represents the "tag", that is an image that is named with the date and git commit and will never be changed -DOCKER_TAG_IMAGE_VERSION="${BRANCH}-${DATE}-${CIRCLE_SHA1}" -DOCKER_TAG_IMAGE_NAME="${DOCKER_PROJECT}:${DOCKER_TAG_IMAGE_VERSION}" - -# build image and login to docker hub -docker build -t $DOCKER_PROJECT . -docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - -# copy the image to each of the two tags, and push -docker tag $DOCKER_PROJECT $DOCKER_BRANCH_IMAGE_NAME -docker tag $DOCKER_PROJECT $DOCKER_TAG_IMAGE_NAME -docker push $DOCKER_BRANCH_IMAGE_NAME -docker push $DOCKER_TAG_IMAGE_NAME From 3360640c434879a19c873010ec2e8022288c38e7 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 30 Aug 2018 23:12:36 +0300 Subject: [PATCH 08/17] Only run semantic-release on production branch --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5ab053b3..3dd2c224 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,3 +19,4 @@ jobs: - stage: release node_js: 10 script: curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash - + if: branch = production From fe69061c0c9ad39929adeb60938be29808692a41 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 30 Aug 2018 23:50:07 +0300 Subject: [PATCH 09/17] Update Docker version This fixes an issue with the version of Alpine we're using: --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8fa41df..3ed292fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: build: working_directory: /app docker: - - image: docker:17.05.0-ce-git + - image: docker:18.06.1-ce-git steps: - checkout - setup_remote_docker From 1753eb5a9cb55e2e8e64c142034a58fceebc81e2 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 4 Sep 2018 10:21:52 +0300 Subject: [PATCH 10/17] feat(docker): Run API processes using pelias user This is enabled by the work in https://github.com/pelias/baseimage/pull/2 and lets us ensure the API is run as a non-root user. Additionally the maintainer label is now set in the baseimage as of (https://github.com/pelias/baseimage/pull/8), so it can be removed here. --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc7836e0..d2a4ba13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ # base image FROM pelias/baseimage -RUN useradd -ms /bin/bash pelias USER pelias -# maintainer information -LABEL maintainer="pelias.team@gmail.com" - # Where the app is built and run inside the docker fs ENV WORK=/home/pelias WORKDIR ${WORK} From e588b92c5ee794fd8808f8817a4c6729bfbe76a5 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sat, 8 Sep 2018 10:58:05 -0400 Subject: [PATCH 11/17] feat(deps): Remove uglify devDependency As I recall, this was only added to mitigate issues with running `npm ls` and correctly showing all dependencies installed (see https://github.com/pelias/api/issues/1179). --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 80521db8..dfbd5c3a 100644 --- a/package.json +++ b/package.json @@ -78,8 +78,7 @@ "source-map": "^0.7.0", "tap-dot": "^2.0.0", "tape": "^4.5.1", - "tmp": "^0.0.33", - "uglify-js": "^3.0.4" + "tmp": "^0.0.33" }, "pre-commit": [ "lint", From 3ad9c342c50403c5cb9956896fc012eb1accb1b5 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Sat, 8 Sep 2018 10:59:53 -0400 Subject: [PATCH 12/17] feat(deps): remove nsp dev dependency The NSP service is shutting down, and as far as I know we have not used this tool frequently at all. NPM has better tools built in now, like `npm audit`. --- package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.json b/package.json index dfbd5c3a..c4effc0d 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "main": "index.js", "bin": "./bin/start", "scripts": { - "audit": "npm shrinkwrap; node node_modules/nsp/bin/nsp check; rm npm-shrinkwrap.json;", "ciao": "node node_modules/ciao/bin/ciao -c test/ciao.json test/ciao", "coverage": "node_modules/.bin/istanbul cover test/unit/run.js", "docs": "./bin/generate-docs", @@ -71,7 +70,6 @@ "difflet": "^1.0.1", "istanbul": "^0.4.2", "jshint": "^2.5.6", - "nsp": "^3.0.0", "pelias-mock-logger": "^1.3.0", "precommit-hook": "^3.0.0", "proxyquire": "^2.0.0", From 8f159ceda7a25a9445dac8377271089190cbbdd8 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Wed, 12 Sep 2018 18:26:31 -0400 Subject: [PATCH 13/17] chore(tests): Update test running script with latest info Use npx instead of calling binaries in node_modules, and link to relevant issue. Connects https://github.com/pelias/pelias/issues/744 --- bin/units | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/units b/bin/units index b011e2ee..fa0ea072 100755 --- a/bin/units +++ b/bin/units @@ -1,5 +1,7 @@ #!/bin/bash +# run tests with pipefail to avoid false passes +# see https://github.com/pelias/pelias/issues/744 set -euo pipefail -node test/unit/run.js | ./node_modules/.bin/tap-dot +node test/unit/run.js | npx tap-dot From 4d9ee0053bfd61fc1da4189b394a716c47acc718 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 19 Sep 2018 18:24:35 +0200 Subject: [PATCH 14/17] feat(libpostal): patch parser output when confused by directionals --- controller/libpostal.js | 30 +++++++++++++++++ test/unit/controller/libpostal.js | 56 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/controller/libpostal.js b/controller/libpostal.js index dd85e9a2..70a2cc1e 100644 --- a/controller/libpostal.js +++ b/controller/libpostal.js @@ -70,6 +70,7 @@ function setup(libpostalService, should_execute) { const initialTime = debugLog.beginTimer(req); libpostalService(req, (err, response) => { + if (err) { // push err.message or err onto req.errors req.errors.push( _.get(err, 'message', err) ); @@ -82,6 +83,10 @@ function setup(libpostalService, should_execute) { return next(); } else { + + // apply fixes for known bugs in libpostal + response = patchBuggyResponses(response); + req.clean.parser = 'libpostal'; req.clean.parsed_text = response.reduce(function(o, f) { if (field_mapping.hasOwnProperty(f.label)) { @@ -109,4 +114,29 @@ function setup(libpostalService, should_execute) { return controller; } +const DIAGONAL_DIRECTIONALS = ['ne','nw','se','sw']; + +// apply fixes for known bugs in libpostal +function patchBuggyResponses(response){ + if( !Array.isArray(response) || !response.length ){ return response; } + + // known bug where the street name is only a directional, in this case we will merge it + // with the subsequent element. + // note: the bug only affects diagonals, not N,S,E,W + // https://github.com/OpenTransitTools/trimet-mod-pelias/issues/20#issuecomment-417732128 + for( var i=0; i { }; +module.exports.tests.bug_fixes = (test, common) => { + test('bug fix: incorrect parsing of diagonal directionals', t => { + const service = (req, callback) => { + const response =[ + { + 'label': 'house_number', + 'value': '4004' + }, + { + 'label': 'road', + 'value': 'nw' + }, + { + 'label': 'suburb', + 'value': 'beaverton-hillsdale' + }, + { + 'label': 'city', + 'value': 'portland' + } + ]; + + callback(null, response); + }; + + const controller = libpostal(service, () => true); + + const req = { + clean: { + text: 'original query' + }, + errors: [] + }; + + controller(req, undefined, () => { + t.deepEquals(req, { + clean: { + text: 'original query', + parser: 'libpostal', + parsed_text: { + number: '4004', + street: 'nw beaverton-hillsdale', + city: 'portland' + } + }, + errors: [] + }, 'req should not have been modified'); + + t.end(); + + }); + + }); + +}; + module.exports.all = (tape, common) => { function test(name, testFunction) { From 192ffc13ddb3fa7dc4afc0cb367e5fe7c8bff30f Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 20 Sep 2018 14:24:01 -0400 Subject: [PATCH 15/17] feat(docs): Add examples of using service `timeout` and `retry` settings These examples have been possible from the beginning, but not explicitly called out in documentation. Fixes https://github.com/pelias/wof-admin-lookup/issues/192 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d90f834c..60384aea 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,9 @@ A good starting configuration file includes this section (fill in the service an "url": "http://libpostal:8080" }, "pip": { - "url": "http://pip-service:4200" + "url": "http://pip-service:4200", + "timeout": 1000, + "retries": 2 }, "interpolation": { "url": "http://interpolation:4300" @@ -77,6 +79,7 @@ A good starting configuration file includes this section (fill in the service an "level": "debug" } } +The `timeout` and `retry` values, as showin in the `pip` service section, are optional but configurable for all services (see [pelias/microservice-wrapper](https://github.com/pelias/microservice-wrapper) for more details). ``` From 47f3f87e72c38a105d478c4a6d6c150c1f3630d6 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 24 Sep 2018 21:27:46 -0400 Subject: [PATCH 16/17] Fix README formatting --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 60384aea..5f6c7864 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,10 @@ A good starting configuration file includes this section (fill in the service an "level": "debug" } } -The `timeout` and `retry` values, as showin in the `pip` service section, are optional but configurable for all services (see [pelias/microservice-wrapper](https://github.com/pelias/microservice-wrapper) for more details). ``` +The `timeout` and `retry` values, as showin in the `pip` service section, are optional but configurable for all services (see [pelias/microservice-wrapper](https://github.com/pelias/microservice-wrapper) for more details). + ## Contributing From 6d7726ae42b7a7b6486d2017ed3b02ba75ae9584 Mon Sep 17 00:00:00 2001 From: Nuno Patronilo Date: Tue, 2 Oct 2018 13:41:24 +0100 Subject: [PATCH 17/17] Parse order on the label field for Portugal Added PRT to the array on var flipNumberAndStreetCountries --- middleware/localNamingConventions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 491cc0ed..38b77d65 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -2,7 +2,7 @@ const check = require('check-types'); const _ = require('lodash'); const field = require('../helper/fieldValue'); -var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL', 'CZE']; +var flipNumberAndStreetCountries = ['DEU', 'FIN', 'SWE', 'NOR', 'DNK', 'ISL', 'CZE','PRT']; function setup() { var api = require('pelias-config').generate().api;