From 63558eb15dbf1f6fb98d5dc5ab42ef6545fe65cd Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Thu, 24 Mar 2016 12:53:48 +0100
Subject: [PATCH 1/4] add ciao dummy data script and README on how to use it

---
 README.md              | 38 +++++++++++++++++++++++++++
 test/ciao_test_data.js | 58 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 test/ciao_test_data.js

diff --git a/README.md b/README.md
index 652f289c..53074f58 100644
--- a/README.md
+++ b/README.md
@@ -34,3 +34,41 @@ The API recognizes the following properties under the top-level `api` key in you
 
 Please fork and pull request against upstream master on a feature branch. Pretty please; provide unit tests and script
 fixtures in the `test` directory.
+
+## Unit tests
+
+You can run the unit test suite using the command:
+
+```bash
+$ npm test
+```
+
+## HTTP tests
+
+We have another set of tests which are used to test the HTTP API layer, these tests send expected HTTP requests and then
+assert that the responses coming back have the correct geoJSON format and HTTP status codes.
+
+You can run the HTTP test suite using the command:
+
+```bash
+$ npm run ciao
+```
+
+Note: some of the tests in this suite fail when no data is present in the index, there is a small set of test documents
+provided in `./test/ciao_test_data` which can be inserted in order to avoid these errors.
+
+To inject dummy data in to your local index:
+
+```bash
+$ node test/ciao_test_data.js
+```
+
+You can confirm the dummy data has been inserted with the command:
+
+```bash
+$ curl localhost:9200/pelias/_count?pretty
+{
+  "count" : 9,
+  ...
+}
+```
diff --git a/test/ciao_test_data.js b/test/ciao_test_data.js
new file mode 100644
index 00000000..70b329f6
--- /dev/null
+++ b/test/ciao_test_data.js
@@ -0,0 +1,58 @@
+
+/**
+  Test data required by the ciao test suite.
+
+  Some tests will fail when run against an empty index, you can use this script
+  to insert some dummy data in to your index before running the tests.
+
+  note: as this is dummy data, care should be taken in order to make sure these
+  documents don't end up in your production index; for that reason the HTTP port
+  has been hard-coded as port:9200.
+**/
+
+// we use the default config to avoid making calls to
+// a cluster running on a non-standard port.
+var client = require('elasticsearch').Client(),
+    async = require('async'),
+    actions = [];
+
+// add one record per 'type' in order to cause the _default_ mapping
+// to be copied when the new type is created.
+var types = ['venue','address','county','region','county','country','admin0','admin1','admin2'],
+    sources = ['test'],
+    layers = ['geonames'];
+
+// iterate over all types/sources/layers and index a test document
+types.forEach( function( type, i1 ){
+  sources.forEach( function( source, i2 ){
+    layers.forEach( function( layer, i3 ){
+      actions.push( function( done ){
+        client.index({
+          index: 'pelias',
+          type: type,
+          id: [i1,i2,i3].join(':'),
+          body: {
+            source: source,
+            layer: layer,
+            name: { default: 'test' },
+            phrase: { default: 'test' },
+            parent: {
+              country_a: ['USA']
+            }
+          }
+        });
+        done();
+      });
+    });
+  });
+});
+
+// call refresh so the index merges the changes
+actions.push( function( done ){
+  client.indices.refresh( { index: 'pelias' }, done);
+});
+
+// perform all actions in series
+async.series( actions, function( err, resp ){
+  console.log('test data inported');
+});

From 8431763a122736a571fbacc468cc92721e88e051 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Thu, 24 Mar 2016 13:15:26 +0100
Subject: [PATCH 2/4] fix handling of ?size= param for autocomplete

---
 sanitiser/_size.js                            | 60 +++++++++++--------
 sanitiser/autocomplete.js                     |  2 +-
 sanitiser/reverse.js                          |  2 +-
 sanitiser/search.js                           |  2 +-
 .../ciao/autocomplete/size_not_default.coffee | 34 +++++++++++
 5 files changed, 71 insertions(+), 29 deletions(-)
 create mode 100644 test/ciao/autocomplete/size_not_default.coffee

diff --git a/sanitiser/_size.js b/sanitiser/_size.js
index 3fb8115b..b2b5e62e 100644
--- a/sanitiser/_size.js
+++ b/sanitiser/_size.js
@@ -5,32 +5,40 @@ var MIN_SIZE = 1,
     DEFAULT_SIZE = 10;
 
 // validate inputs, convert types and apply defaults
-function sanitize( raw, clean ){
-
-  // error & warning messages
-  var messages = { errors: [], warnings: [] };
-
-  // coercions
-  clean.size = parseInt( raw.size, 10 );
-
-  // invalid numeric input
-  if( isNaN( clean.size ) ){
-    clean.size = DEFAULT_SIZE;
-  }
-  // ensure size falls within defined range
-  else if( clean.size > MAX_SIZE ){
-    // set the max size
-    messages.warnings.push('out-of-range integer \'size\', using MAX_SIZE');
-    clean.size = MAX_SIZE;
-  }
-  else if( clean.size < MIN_SIZE ){
-    // set the min size
-    messages.warnings.push('out-of-range integer \'size\', using MIN_SIZE');
-    clean.size = MIN_SIZE;
-  }
-
-  return messages;
+function setup( size_min, size_max, size_def ){
+
+  // allow caller to inject custom min/max/default values
+  if( !check.number( size_min ) ){ size_min = MIN_SIZE; }
+  if( !check.number( size_max ) ){ size_max = MAX_SIZE; }
+  if( !check.number( size_def ) ){ size_def = DEFAULT_SIZE; }
+
+  return function sanitize( raw, clean ){
+
+    // error & warning messages
+    var messages = { errors: [], warnings: [] };
+
+    // coercions
+    clean.size = parseInt( raw.size, 10 );
+
+    // invalid numeric input
+    if( isNaN( clean.size ) ){
+      clean.size = size_def;
+    }
+    // ensure size falls within defined range
+    else if( clean.size > size_max ){
+      // set the max size
+      messages.warnings.push('out-of-range integer \'size\', using MAX_SIZE');
+      clean.size = size_max;
+    }
+    else if( clean.size < size_min ){
+      // set the min size
+      messages.warnings.push('out-of-range integer \'size\', using MIN_SIZE');
+      clean.size = size_min;
+    }
+
+    return messages;
+  };
 }
 
 // export function
-module.exports = sanitize;
+module.exports = setup;
diff --git a/sanitiser/autocomplete.js b/sanitiser/autocomplete.js
index c6dc08da..7bb621c1 100644
--- a/sanitiser/autocomplete.js
+++ b/sanitiser/autocomplete.js
@@ -2,7 +2,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
-      size: require('../sanitiser/_size'),
+      size: require('../sanitiser/_size')(10, 10, 10),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_autocomplete: require('../sanitiser/_geo_autocomplete'),
     };
diff --git a/sanitiser/reverse.js b/sanitiser/reverse.js
index d707fcea..fabfcf65 100644
--- a/sanitiser/reverse.js
+++ b/sanitiser/reverse.js
@@ -7,7 +7,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
       sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitisers, must be run after them
       sources_and_layers: require('../sanitiser/_sources_and_layers'),
-      size: require('../sanitiser/_size'),
+      size: require('../sanitiser/_size')(/* use defaults*/),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_reverse: require('../sanitiser/_geo_reverse'),
       boundary_country: require('../sanitiser/_boundary_country'),
diff --git a/sanitiser/search.js b/sanitiser/search.js
index 980b5a04..c9c68330 100644
--- a/sanitiser/search.js
+++ b/sanitiser/search.js
@@ -4,7 +4,7 @@ var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
-      size: require('../sanitiser/_size'),
+      size: require('../sanitiser/_size')(/* use defaults*/),
       layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
       sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
       // depends on the layers and sources sanitisers, must be run after them
diff --git a/test/ciao/autocomplete/size_not_default.coffee b/test/ciao/autocomplete/size_not_default.coffee
new file mode 100644
index 00000000..8baf233d
--- /dev/null
+++ b/test/ciao/autocomplete/size_not_default.coffee
@@ -0,0 +1,34 @@
+
+#> set size (autocomplete does not allow size to be changed)
+path: '/v1/autocomplete?text=a&size=3'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.exist json.geocoding.warnings
+json.geocoding.warnings.should.eql [ 'out-of-range integer \'size\', using MIN_SIZE' ]
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10 # should remain the default size

From 00b123adfbec4f362c66772ea5d0e3c66b3e0d42 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Thu, 24 Mar 2016 13:22:03 +0100
Subject: [PATCH 3/4] enable ?source=s and ?layers params for autocomplete

---
 sanitiser/autocomplete.js                     |  6 +++
 .../autocomplete/layers_alias_address.coffee  | 34 ++++++++++++++
 .../autocomplete/layers_alias_coarse.coffee   | 46 +++++++++++++++++++
 test/ciao/autocomplete/layers_invalid.coffee  | 34 ++++++++++++++
 .../layers_mix_invalid_valid.coffee           | 35 ++++++++++++++
 test/ciao/autocomplete/layers_multiple.coffee | 34 ++++++++++++++
 test/ciao/autocomplete/layers_single.coffee   | 34 ++++++++++++++
 test/ciao/autocomplete/sources_invalid.coffee | 34 ++++++++++++++
 .../sources_layers_invalid_combo.coffee       | 37 +++++++++++++++
 .../sources_layers_valid_combo.coffee         | 35 ++++++++++++++
 .../ciao/autocomplete/sources_multiple.coffee | 34 ++++++++++++++
 test/ciao/autocomplete/sources_single.coffee  | 34 ++++++++++++++
 12 files changed, 397 insertions(+)
 create mode 100644 test/ciao/autocomplete/layers_alias_address.coffee
 create mode 100644 test/ciao/autocomplete/layers_alias_coarse.coffee
 create mode 100644 test/ciao/autocomplete/layers_invalid.coffee
 create mode 100644 test/ciao/autocomplete/layers_mix_invalid_valid.coffee
 create mode 100644 test/ciao/autocomplete/layers_multiple.coffee
 create mode 100644 test/ciao/autocomplete/layers_single.coffee
 create mode 100644 test/ciao/autocomplete/sources_invalid.coffee
 create mode 100644 test/ciao/autocomplete/sources_layers_invalid_combo.coffee
 create mode 100644 test/ciao/autocomplete/sources_layers_valid_combo.coffee
 create mode 100644 test/ciao/autocomplete/sources_multiple.coffee
 create mode 100644 test/ciao/autocomplete/sources_single.coffee

diff --git a/sanitiser/autocomplete.js b/sanitiser/autocomplete.js
index 7bb621c1..f9698956 100644
--- a/sanitiser/autocomplete.js
+++ b/sanitiser/autocomplete.js
@@ -1,8 +1,14 @@
+var type_mapping = require('../helper/type_mapping');
+
 var sanitizeAll = require('../sanitiser/sanitizeAll'),
     sanitizers = {
       singleScalarParameters: require('../sanitiser/_single_scalar_parameters'),
       text: require('../sanitiser/_text'),
       size: require('../sanitiser/_size')(10, 10, 10),
+      layers: require('../sanitiser/_targets')('layers', type_mapping.layer_mapping),
+      sources: require('../sanitiser/_targets')('sources', type_mapping.source_mapping),
+      // depends on the layers and sources sanitisers, must be run after them
+      sources_and_layers: require('../sanitiser/_sources_and_layers'),
       private: require('../sanitiser/_flag_bool')('private', false),
       geo_autocomplete: require('../sanitiser/_geo_autocomplete'),
     };
diff --git a/test/ciao/autocomplete/layers_alias_address.coffee b/test/ciao/autocomplete/layers_alias_address.coffee
new file mode 100644
index 00000000..d393971a
--- /dev/null
+++ b/test/ciao/autocomplete/layers_alias_address.coffee
@@ -0,0 +1,34 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=address'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["address"]
diff --git a/test/ciao/autocomplete/layers_alias_coarse.coffee b/test/ciao/autocomplete/layers_alias_coarse.coffee
new file mode 100644
index 00000000..4a359bdd
--- /dev/null
+++ b/test/ciao/autocomplete/layers_alias_coarse.coffee
@@ -0,0 +1,46 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=coarse'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql [ "continent",
+  "macrocountry",
+  "country",
+  "dependency",
+  "region",
+  "locality",
+  "localadmin",
+  "county",
+  "macrohood",
+  "neighbourhood",
+  "microhood",
+  "disputed"
+]
diff --git a/test/ciao/autocomplete/layers_invalid.coffee b/test/ciao/autocomplete/layers_invalid.coffee
new file mode 100644
index 00000000..b4ce0565
--- /dev/null
+++ b/test/ciao/autocomplete/layers_invalid.coffee
@@ -0,0 +1,34 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=notlayer'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
diff --git a/test/ciao/autocomplete/layers_mix_invalid_valid.coffee b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee
new file mode 100644
index 00000000..8bac1ccf
--- /dev/null
+++ b/test/ciao/autocomplete/layers_mix_invalid_valid.coffee
@@ -0,0 +1,35 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=country,notlayer'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'notlayer\' is an invalid layers parameter. Valid options: coarse,address,venue,country,region,county,locality,continent,macrocountry,dependency,localadmin,macrohood,neighbourhood,microhood,disputed' ]
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+should.not.exist json.geocoding.query['layers']
diff --git a/test/ciao/autocomplete/layers_multiple.coffee b/test/ciao/autocomplete/layers_multiple.coffee
new file mode 100644
index 00000000..fed6f6f7
--- /dev/null
+++ b/test/ciao/autocomplete/layers_multiple.coffee
@@ -0,0 +1,34 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=country,region'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["country","region"]
diff --git a/test/ciao/autocomplete/layers_single.coffee b/test/ciao/autocomplete/layers_single.coffee
new file mode 100644
index 00000000..fdae272a
--- /dev/null
+++ b/test/ciao/autocomplete/layers_single.coffee
@@ -0,0 +1,34 @@
+
+#> layer alias
+path: '/v1/autocomplete?text=a&layers=country'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["country"]
diff --git a/test/ciao/autocomplete/sources_invalid.coffee b/test/ciao/autocomplete/sources_invalid.coffee
new file mode 100644
index 00000000..38a12162
--- /dev/null
+++ b/test/ciao/autocomplete/sources_invalid.coffee
@@ -0,0 +1,34 @@
+
+#> sources filter
+path: '/v1/autocomplete?text=a&sources=openstreetmap,notasource'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ '\'notasource\' is an invalid sources parameter. Valid options: osm,oa,gn,wof,openstreetmap,openaddresses,geonames,whosonfirst' ]
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
diff --git a/test/ciao/autocomplete/sources_layers_invalid_combo.coffee b/test/ciao/autocomplete/sources_layers_invalid_combo.coffee
new file mode 100644
index 00000000..3d27a200
--- /dev/null
+++ b/test/ciao/autocomplete/sources_layers_invalid_combo.coffee
@@ -0,0 +1,37 @@
+
+#> sources and layers specified (invalid combo)
+path: '/v1/autocomplete?text=a&sources=whosonfirst&layers=address'
+
+#? 200 ok
+response.statusCode.should.be.equal 400
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.exist json.geocoding.errors
+json.geocoding.errors.should.eql [ 'You have specified both the `sources` and `layers` parameters in a combination that will return no results: the whosonfirst source has nothing in the address layer' ]
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["whosonfirst"]
+should.not.exist json.geocoding.query['type']
diff --git a/test/ciao/autocomplete/sources_layers_valid_combo.coffee b/test/ciao/autocomplete/sources_layers_valid_combo.coffee
new file mode 100644
index 00000000..2779009f
--- /dev/null
+++ b/test/ciao/autocomplete/sources_layers_valid_combo.coffee
@@ -0,0 +1,35 @@
+
+#> sources and layers specified
+path: '/v1/autocomplete?text=a&sources=openaddresses&layers=address'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.layers.should.eql ["address"]
+json.geocoding.query.sources.should.eql ["openaddresses"]
diff --git a/test/ciao/autocomplete/sources_multiple.coffee b/test/ciao/autocomplete/sources_multiple.coffee
new file mode 100644
index 00000000..5ddad82f
--- /dev/null
+++ b/test/ciao/autocomplete/sources_multiple.coffee
@@ -0,0 +1,34 @@
+
+#> sources filter
+path: "/v1/autocomplete?text=a&sources=openstreetmap,geonames"
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header "charset", 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.sources.should.eql ["openstreetmap", "geonames"]
diff --git a/test/ciao/autocomplete/sources_single.coffee b/test/ciao/autocomplete/sources_single.coffee
new file mode 100644
index 00000000..d1df9c9d
--- /dev/null
+++ b/test/ciao/autocomplete/sources_single.coffee
@@ -0,0 +1,34 @@
+
+#> sources filter
+path: '/v1/autocomplete?text=a&sources=openstreetmap'
+
+#? 200 ok
+response.statusCode.should.be.equal 200
+response.should.have.header 'charset', 'utf8'
+response.should.have.header 'content-type', 'application/json; charset=utf-8'
+
+#? valid geocoding block
+should.exist json.geocoding
+should.exist json.geocoding.version
+should.exist json.geocoding.attribution
+should.exist json.geocoding.query
+should.exist json.geocoding.engine
+should.exist json.geocoding.engine.name
+should.exist json.geocoding.engine.author
+should.exist json.geocoding.engine.version
+should.exist json.geocoding.timestamp
+
+#? valid geojson
+json.type.should.be.equal 'FeatureCollection'
+json.features.should.be.instanceof Array
+
+#? expected errors
+should.not.exist json.geocoding.errors
+
+#? expected warnings
+should.not.exist json.geocoding.warnings
+
+#? inputs
+json.geocoding.query['text'].should.eql 'a'
+json.geocoding.query['size'].should.eql 10
+json.geocoding.query.sources.should.eql ["openstreetmap"]

From 0abda099a6722cde040c60251ed7310f5e3dc193 Mon Sep 17 00:00:00 2001
From: Peter Johnson <insomnia@rcpt.at>
Date: Thu, 24 Mar 2016 14:49:20 +0100
Subject: [PATCH 4/4] bugfixes and more tests

---
 query/autocomplete.js                         |  8 ++
 .../autocomplete_with_source_filtering.js     | 85 +++++++++++++++++
 .../fixture/reverse_with_source_filtering.js  | 51 ++++++++++
 .../fixture/search_with_source_filtering.js   | 93 +++++++++++++++++++
 test/unit/query/autocomplete.js               | 13 +++
 test/unit/query/search.js                     | 13 +++
 test/unit/sanitiser/_size.js                  |  8 +-
 test/unit/sanitiser/autocomplete.js           |  2 +-
 8 files changed, 268 insertions(+), 5 deletions(-)
 create mode 100644 test/unit/fixture/autocomplete_with_source_filtering.js
 create mode 100644 test/unit/fixture/reverse_with_source_filtering.js
 create mode 100644 test/unit/fixture/search_with_source_filtering.js

diff --git a/query/autocomplete.js b/query/autocomplete.js
index 4bc98c7c..ffc57396 100644
--- a/query/autocomplete.js
+++ b/query/autocomplete.js
@@ -41,6 +41,9 @@ query.score( views.focus_selected_layers( views.ngrams_strict ) );
 query.score( peliasQuery.view.popularity( views.ngrams_strict ) );
 query.score( peliasQuery.view.population( views.ngrams_strict ) );
 
+// non-scoring hard filters
+query.filter( peliasQuery.view.sources );
+
 // --------------------------------
 
 /**
@@ -51,6 +54,11 @@ function generateQuery( clean ){
 
   var vs = new peliasQuery.Vars( defaults );
 
+  // sources
+  if( check.array(clean.sources) && clean.sources.length ){
+    vs.var( 'sources', clean.sources );
+  }
+
   // mark the name as incomplete (user has not yet typed a comma)
   vs.var( 'input:name:isComplete', false );
 
diff --git a/test/unit/fixture/autocomplete_with_source_filtering.js b/test/unit/fixture/autocomplete_with_source_filtering.js
new file mode 100644
index 00000000..22c12a5d
--- /dev/null
+++ b/test/unit/fixture/autocomplete_with_source_filtering.js
@@ -0,0 +1,85 @@
+
+module.exports = {
+  'query': {
+    'filtered': {
+      'query': {
+        'bool': {
+          'must': [{
+            'match': {
+              'name.default': {
+                'analyzer': 'peliasPhrase',
+                'boost': 100,
+                'query': 'test',
+                'type': 'phrase',
+                'operator': 'and'
+              }
+            }
+          }],
+          'should':[{
+            'function_score': {
+              'query': {
+                'match': {
+                  'name.default': {
+                    'analyzer': 'peliasPhrase',
+                    'boost': 100,
+                    'query': 'test',
+                    'type': 'phrase',
+                    'operator': 'and'
+                  }
+                }
+              },
+              'max_boost': 20,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'popularity',
+                  'missing': 1
+                },
+                'weight': 1
+              }]
+            }
+          },{
+            'function_score': {
+              'query': {
+                'match': {
+                  'name.default': {
+                    'analyzer': 'peliasPhrase',
+                    'boost': 100,
+                    'query': 'test',
+                    'type': 'phrase',
+                    'operator': 'and'
+                  }
+                }
+              },
+              'max_boost': 20,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'population',
+                  'missing': 1
+                },
+                'weight': 3
+              }]
+            }
+          }]
+        }
+      },
+      'filter': {
+        'bool': {
+          'must': [{
+            'terms': {
+              'source': ['test_source']
+            }
+          }]
+        }
+      }
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 20,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/reverse_with_source_filtering.js b/test/unit/fixture/reverse_with_source_filtering.js
new file mode 100644
index 00000000..28fce18b
--- /dev/null
+++ b/test/unit/fixture/reverse_with_source_filtering.js
@@ -0,0 +1,51 @@
+var vs = require('../../../query/reverse_defaults');
+
+module.exports = {
+  'query': {
+    'filtered': {
+      'query': {
+        'bool': {
+          'must': []
+        }
+      },
+      'filter': {
+        'bool': {
+          'must': [
+            {
+              'geo_distance': {
+                'distance': '500km',
+                'distance_type': 'plane',
+                'optimize_bbox': 'indexed',
+                '_cache': true,
+                'center_point': {
+                  'lat': 29.49136,
+                  'lon': -82.50622
+                }
+              }
+            },
+            {
+              'terms': {
+                'source': ['test']
+              }
+            }
+          ]
+        }
+      }
+    }
+  },
+  'sort': [
+    '_score',
+    {
+      '_geo_distance': {
+        'center_point': {
+          'lat': 29.49136,
+          'lon': -82.50622
+        },
+        'order': 'asc',
+        'distance_type': 'plane'
+      }
+    }
+  ],
+  'size': vs.size,
+  'track_scores': true
+};
diff --git a/test/unit/fixture/search_with_source_filtering.js b/test/unit/fixture/search_with_source_filtering.js
new file mode 100644
index 00000000..593eac5b
--- /dev/null
+++ b/test/unit/fixture/search_with_source_filtering.js
@@ -0,0 +1,93 @@
+
+module.exports = {
+  'query': {
+    'filtered': {
+      'query': {
+        'bool': {
+          'must': [{
+            'match': {
+              'name.default': {
+                'query': 'test',
+                'boost': 1,
+                'analyzer': 'peliasOneEdgeGram'
+              }
+            }
+          }],
+          'should': [{
+            'match': {
+              'phrase.default': {
+                'query': 'test',
+                'analyzer': 'peliasPhrase',
+                'type': 'phrase',
+                'boost': 1,
+                'slop': 2
+              }
+            }
+          },{
+            'function_score': {
+              'query': {
+                'match': {
+                  'phrase.default': {
+                    'query': 'test',
+                    'analyzer': 'peliasPhrase',
+                    'type': 'phrase',
+                    'slop': 2,
+                    'boost': 1
+                  }
+                }
+              },
+              'max_boost': 20,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'popularity',
+                  'missing': 1
+                },
+                'weight': 1
+              }]
+            }
+          },{
+            'function_score': {
+              'query': {
+                'match': {
+                  'phrase.default': {
+                    'query': 'test',
+                    'analyzer': 'peliasPhrase',
+                    'type': 'phrase',
+                    'slop': 2,
+                    'boost': 1
+                  }
+                }
+              },
+              'max_boost': 20,
+              'score_mode': 'first',
+              'boost_mode': 'replace',
+              'functions': [{
+                'field_value_factor': {
+                  'modifier': 'log1p',
+                  'field': 'population',
+                  'missing': 1
+                },
+                'weight': 2
+              }]
+            }
+          }]
+        }
+      },
+      'filter': {
+        'bool': {
+          'must': [{
+            'terms': {
+              'source': ['test_source']
+            }
+          }]
+        }
+      }
+    }
+  },
+  'sort': [ '_score' ],
+  'size': 20,
+  'track_scores': true
+};
diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js
index dc973ddc..0e09457b 100644
--- a/test/unit/query/autocomplete.js
+++ b/test/unit/query/autocomplete.js
@@ -95,6 +95,19 @@ module.exports.tests.query = function(test, common) {
     t.deepEqual(compiled, expected, 'valid autocomplete query');
     t.end();
   });
+
+  test('valid sources filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'sources': ['test_source']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/autocomplete_with_source_filtering');
+
+    t.deepEqual(compiled, expected, 'valid autocomplete query with source filtering');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {
diff --git a/test/unit/query/search.js b/test/unit/query/search.js
index f20122bb..e503311b 100644
--- a/test/unit/query/search.js
+++ b/test/unit/query/search.js
@@ -182,6 +182,19 @@ module.exports.tests.query = function(test, common) {
     t.end();
   });
 
+  test('valid sources filter', function(t) {
+    var query = generate({
+      'text': 'test',
+      'sources': ['test_source']
+    });
+
+    var compiled = JSON.parse( JSON.stringify( query ) );
+    var expected = require('../fixture/search_with_source_filtering');
+
+    t.deepEqual(compiled, expected, 'valid search query with source filtering');
+    t.end();
+  });
+
 };
 
 module.exports.all = function (tape, common) {
diff --git a/test/unit/sanitiser/_size.js b/test/unit/sanitiser/_size.js
index e4e61ac9..67c8bd84 100644
--- a/test/unit/sanitiser/_size.js
+++ b/test/unit/sanitiser/_size.js
@@ -6,7 +6,7 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size=0', function(t) {
     var raw = { size: 0 };
     var clean = {};
-    var res = sanitize(raw, clean);
+    var res = sanitize(/*defaults*/)(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 1, 'should return warning');
     t.equal(res.warnings[0], 'out-of-range integer \'size\', using MIN_SIZE', 'check warning text');
@@ -17,7 +17,7 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size=10000', function(t) {
     var raw = { size: 10000 };
     var clean = {};
-    var res = sanitize(raw, clean);
+    var res = sanitize(/*defaults*/)(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 1, 'should return warning');
     t.equal(res.warnings[0], 'out-of-range integer \'size\', using MAX_SIZE', 'check warning text');
@@ -28,7 +28,7 @@ module.exports.tests.sanitize_size = function(test, common) {
   test('size not set', function(t) {
     var raw = {};
     var clean = {};
-    var res = sanitize(raw, clean);
+    var res = sanitize(/*defaults*/)(raw, clean);
     t.equal(res.errors.length, 0, 'should return no errors');
     t.equal(res.warnings.length, 0, 'should return no warning');
     t.equal(clean.size, 10, 'default to 10');
@@ -41,7 +41,7 @@ module.exports.tests.sanitize_size = function(test, common) {
     test('size=' + size, function (t) {
       var raw = {size: size};
       var clean = {};
-      var res = sanitize(raw, clean);
+      var res = sanitize(/*defaults*/)(raw, clean);
       t.equal(res.errors.length, 0, 'should return no errors');
       t.equal(res.warnings.length, 0, 'should return warning');
       t.equal(clean.size, 5, 'set to correct integer');
diff --git a/test/unit/sanitiser/autocomplete.js b/test/unit/sanitiser/autocomplete.js
index 4044d7f1..26bf9afb 100644
--- a/test/unit/sanitiser/autocomplete.js
+++ b/test/unit/sanitiser/autocomplete.js
@@ -4,7 +4,7 @@ module.exports.tests = {};
 
 module.exports.tests.sanitisers = function(test, common) {
   test('check sanitiser list', function (t) {
-    var expected = ['singleScalarParameters', 'text', 'size', 'private', 'geo_autocomplete' ];
+    var expected = ['singleScalarParameters', 'text', 'size', 'layers', 'sources', 'sources_and_layers', 'private', 'geo_autocomplete' ];
     t.deepEqual(Object.keys(autocomplete.sanitiser_list), expected);
     t.end();
   });