Add CSV importer

cmd/download.sh

@@ -7,12 +7,14 @@ function download_oa(){ compose_run 'openaddresses' './bin/download'; }
 function download_osm(){ compose_run 'openstreetmap' './bin/download'; }
 function download_tiger(){ compose_run 'interpolation' './bin/download-tiger'; }
 function download_transit(){ compose_run 'transit' './bin/download'; }
+function download_csv(){ compose_run 'csv-importer' './bin/download'; }
 
 register 'download' 'wof' '(re)download whosonfirst data' download_wof
 register 'download' 'oa' '(re)download openaddresses data' download_oa
 register 'download' 'osm' '(re)download openstreetmap data' download_osm
 register 'download' 'tiger' '(re)download TIGER data' download_tiger
 register 'download' 'transit' '(re)download transit data' download_transit
+register 'download' 'csv' '(re)download csv data' download_csv
 
 # download all the data to be used by imports
 function download_all(){
@@ -21,6 +23,7 @@ function download_all(){
   download_osm &
   download_tiger &
   download_transit &
+  download_csv &
   wait
 }
 

cmd/import.sh

@@ -7,12 +7,14 @@ function import_oa(){ compose_run 'openaddresses' './bin/start'; }
 function import_osm(){ compose_run 'openstreetmap' './bin/start'; }
 function import_polylines(){ compose_run 'polylines' './bin/start'; }
 function import_transit(){ compose_run 'transit' './bin/start'; }
+function import_csv(){ compose_run 'csv-importer' './bin/start'; }
 
 register 'import' 'wof' '(re)import whosonfirst data' import_wof
 register 'import' 'oa' '(re)import openaddresses data' import_oa
 register 'import' 'osm' '(re)import openstreetmap data' import_osm
 register 'import' 'polylines' '(re)import polylines data' import_polylines
 register 'import' 'transit' '(re)import transit data' import_transit
+register 'import' 'csv' '(re)import csv data' import_csv
 
 # import all the data to be used by imports
 # note: running importers in parallel can cause issues due to high CPU & RAM requirements.
@@ -22,6 +24,7 @@ function import_all(){
   import_osm
   import_polylines
   import_transit
+  import_csv
 }
 
 register 'import' 'all' '(re)import all data' import_all

projects/portland-metro/docker-compose.yml

@@ -67,6 +67,14 @@ services:
     volumes:
       - "./pelias.json:/code/pelias.json"
       - "${DATA_DIR}:/data"
+  csv-importer:
+    image: pelias/csv-importer:master
+    container_name: pelias_csv_importer
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
   polylines:
     image: pelias/polylines:master
     container_name: pelias_polylines

projects/portland-metro/pelias.json

@@ -44,6 +44,13 @@
         "/data/blacklist/osm.txt"
       ]
     },
+    "csv": {
+      "datapath": "/data/csv",
+      "files": [],
+      "download": [
+        "https://raw.githubusercontent.com/pelias/transit/master/test/landmarks.csv"
+      ]
+    },
     "geonames": {
       "datapath": "/data/geonames",
       "countryCode": "ALL"