From e2316b240a4f84ad31c789419365cbe0e39b654c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Thu, 20 Sep 2018 19:34:15 -0400 Subject: [PATCH] Add Australia project Australia is a great mid-sized country that is easy to run on a personal computer but has a decent amount of records, around 15 million. --- projects/australia/.env | 3 + projects/australia/README.md | 36 ++++++ projects/australia/blacklist/osm.txt | 0 projects/australia/docker-compose.yml | 118 ++++++++++++++++++ projects/australia/pelias.json | 86 +++++++++++++ .../australia/test_cases/search_address.json | 16 +++ .../australia/test_cases/search_venue.json | 49 ++++++++ 7 files changed, 308 insertions(+) create mode 100644 projects/australia/.env create mode 100644 projects/australia/README.md create mode 100644 projects/australia/blacklist/osm.txt create mode 100644 projects/australia/docker-compose.yml create mode 100644 projects/australia/pelias.json create mode 100644 projects/australia/test_cases/search_address.json create mode 100644 projects/australia/test_cases/search_venue.json diff --git a/projects/australia/.env b/projects/australia/.env new file mode 100644 index 0000000..5182d43 --- /dev/null +++ b/projects/australia/.env @@ -0,0 +1,3 @@ +COMPOSE_PROJECT_NAME=pelias +DATA_DIR=/tmp/pelias/australia +DOCKER_USER=1000 diff --git a/projects/australia/README.md b/projects/australia/README.md new file mode 100644 index 0000000..3dd17ea --- /dev/null +++ b/projects/australia/README.md @@ -0,0 +1,36 @@ + +# Australia + +This project is configured to download/prepare/build a complete Pelias installation for the country of Australia. + +It's designed to be a larger example of using Pelias. + +# Setup + +Please refer to the instructions at https://github.com/pelias/docker in order to install and configure your docker environment. + +The minimum configuration required in order to run this project are [installing prerequisites](https://github.com/pelias/docker#prerequisites), [install the pelias command](https://github.com/pelias/docker#installing-the-pelias-command) and [configure the environment](https://github.com/pelias/docker#configure-environment). + +Please ensure that's all working fine before continuing. + +# Run a Build + +To run a complete build, execute the following commands: + +```bash +pelias compose pull +pelias elastic start +pelias elastic wait +pelias elastic create +pelias download all +pelias prepare all +pelias import all +pelias compose up +pelias test run +``` + +# Make an Example Query + +You can now make queries against your new Pelias build: + +http://localhost:4000/v1/search?text=melbourne diff --git a/projects/australia/blacklist/osm.txt b/projects/australia/blacklist/osm.txt new file mode 100644 index 0000000..e69de29 diff --git a/projects/australia/docker-compose.yml b/projects/australia/docker-compose.yml new file mode 100644 index 0000000..7a49a30 --- /dev/null +++ b/projects/australia/docker-compose.yml @@ -0,0 +1,118 @@ +version: '3' +networks: + default: + driver: bridge +services: + libpostal: + image: pelias/libpostal-service + container_name: pelias_libpostal + user: "${DOCKER_USER}" + restart: always + ports: [ "4400:4400" ] + schema: + image: pelias/schema:master + container_name: pelias_schema + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + api: + image: pelias/api:master + container_name: pelias_api + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4000" ] + ports: [ "4000:4000" ] + volumes: + - "./pelias.json:/code/pelias.json" + placeholder: + image: pelias/placeholder:master + container_name: pelias_placeholder + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4100" ] + ports: [ "4100:4100" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + whosonfirst: + image: pelias/whosonfirst:master + container_name: pelias_whosonfirst + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + - "./blacklist/:/data/blacklist" + openstreetmap: + image: pelias/openstreetmap:master + container_name: pelias_openstreetmap + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + - "./blacklist/:/data/blacklist" + openaddresses: + image: pelias/openaddresses:master + container_name: pelias_openaddresses + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + - "./blacklist/:/data/blacklist" + transit: + image: pelias/transit:master + container_name: pelias_transit + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + polylines: + image: pelias/polylines:master + container_name: pelias_polylines + user: "${DOCKER_USER}" + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + interpolation: + image: pelias/interpolation:master + container_name: pelias_interpolation + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4300" ] + ports: [ "4300:4300" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + pip: + image: pelias/pip-service:master + container_name: pelias_pip-service + user: "${DOCKER_USER}" + restart: always + environment: [ "PORT=4200" ] + ports: [ "4200:4200" ] + volumes: + - "./pelias.json:/code/pelias.json" + - "${DATA_DIR}:/data" + elasticsearch: + image: pelias/elasticsearch + container_name: pelias_elasticsearch + restart: always + ports: [ "9200:9200", "9300:9300" ] + volumes: + - "${DATA_DIR}/elasticsearch:/usr/share/elasticsearch/data" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + cap_add: [ "IPC_LOCK" ] + fuzzy-tester: + image: pelias/fuzzy-tester:master + container_name: pelias_fuzzy_tester + user: "${DOCKER_USER}" + restart: "no" + command: "--help" + volumes: + - "./pelias.json:/code/pelias.json" + - "./test_cases:/code/pelias/fuzzy-tester/test_cases" diff --git a/projects/australia/pelias.json b/projects/australia/pelias.json new file mode 100644 index 0000000..d1c0af5 --- /dev/null +++ b/projects/australia/pelias.json @@ -0,0 +1,86 @@ +{ + "logger": { + "level": "info", + "timestamp": false + }, + "esclient": { + "hosts": [ + { "host": "elasticsearch" } + ] + }, + "elasticsearch": { + "settings": { + "index": { + "refresh_interval": "10s", + "number_of_replicas": "0", + "number_of_shards": "3" + } + } + }, + "acceptance-tests": { + "endpoints": { + "docker": "http://api:4000/v1/" + } + }, + "api": { + "textAnalyzer": "libpostal", + "services": { + "placeholder": { "url": "http://placeholder:4100" }, + "pip": { "url": "http://pip:4200" }, + "interpolation": { "url": "http://interpolation:4300" }, + "libpostal": { "url": "http://libpostal:4400" } + } + }, + "imports": { + "adminLookup": { + "enabled": true + }, + "geonames": { + "datapath": "/data/geonames", + "countryCode": "AU" + }, + "openstreetmap": { + "download": [ + { "sourceURL": "http://download.geofabrik.de/australia-oceania/australia-latest.osm.pbf" } + ], + "leveldbpath": "/tmp", + "datapath": "/data/openstreetmap", + "import": [{ + "filename": "australia-latest.osm.pbf" + }] + }, + "openaddresses": { + "datapath": "/data/openaddresses", + "files": [ + "au/countrywide.csv" + ] + }, + "polyline": { + "datapath": "/data/polylines", + "files": [ "extract.0sv" ] + }, + "whosonfirst": { + "datapath": "/data/whosonfirst", + "importVenues": false, + "importPostalcodes": true, + "importPlace": [ + "85632793" + ] + }, + "transit": { + "datapath": "/data/transit", + "feeds": [ + ] + }, + "interpolation": { + "download": { + "tiger": { + "datapath": "/data/tiger", + "states": [ + { "state_code": 53, "county_code": 11, "name": "Clark County, WA" } + ] + } + } + } + } +} diff --git a/projects/australia/test_cases/search_address.json b/projects/australia/test_cases/search_address.json new file mode 100644 index 0000000..2327fc5 --- /dev/null +++ b/projects/australia/test_cases/search_address.json @@ -0,0 +1,16 @@ +{ + "name": "/v1/search address", + "description": "addresses in Portland, OR", + "priorityThresh": 1, + "normalizers": { + "name": [ + "toUpperCase", + "removeOrdinals", + "stripPunctuation", + "abbreviateDirectionals", + "abbreviateStreetSuffixes" + ] + }, + "tests": [ + ] +} diff --git a/projects/australia/test_cases/search_venue.json b/projects/australia/test_cases/search_venue.json new file mode 100644 index 0000000..c4aac0b --- /dev/null +++ b/projects/australia/test_cases/search_venue.json @@ -0,0 +1,49 @@ +{ + "name": "/v1/search venues", + "priorityThresh": 1, + "endpoint": "search", + "tests": [ + { + "id": 1, + "status": "pass", + "notes": "portland international should come up for PDX", + "in": { + "text": "pdx" + }, + "expected": { + "properties": [ + { + "layer": "venue", + "name": "Portland International Airport", + "country_a": "USA", + "country": "United States", + "region": "Oregon", + "region_a": "OR", + "locality": "Portland" + } + ] + } + }, + { + "id": 2, + "status": "pass", + "notes": "portland international should come up for PDX", + "in": { + "text": "pdx airport" + }, + "expected": { + "properties": [ + { + "layer": "venue", + "name": "Portland International Airport", + "country_a": "USA", + "country": "United States", + "region": "Oregon", + "region_a": "OR", + "locality": "Portland" + } + ] + } + } + ] +}