Keep ordered keys as separate property in pivot format returned by convert methods. (#2314)

This fixes the issue where targets order would not be kept when some of
them have numerical values (ex: "1", "2", ..).

spec/data.convert.js

@@ -1,97 +1,392 @@
 import c3 from '../src';
 
 const $$ = c3.chart.internal.fn;
+$$.d3 = require("d3");
 
-describe('$$.convertColumnsToData', () => {
+describe('data.convert', () => {
-    it('converts column data to normalized data', () => {
+
-        const data = $$.convertColumnsToData([
+    describe('$$.convertColumnsToData', () => {
-            ["cat1", "a", "b", "c", "d"],
+        it('converts column data to normalized data', () => {
-            ["data1", 30, 200, 100, 400],
+            const data = $$.convertColumnsToData([
-            ["cat2", "b", "a", "c", "d", "e", "f"],
+                ["cat1", "a", "b", "c", "d"],
-            ["data2", 400, 60, 200, 800, 10, 10]
+                ["data1", 30, 200, 100, 400],
-        ]);
+                ["cat2", "b", "a", "c", "d", "e", "f"],
-
+                ["data2", 400, 60, 200, 800, 10, 10]
-        expect(data).toEqual([{
+            ]);
-            cat1: 'a',
+
-            data1: 30,
+            expect(data).toEqual({
-            cat2: 'b',
+                keys: [ 'cat1', 'data1', 'cat2', 'data2' ],
-            data2: 400
+                rows: [{
-        }, {
+                    cat1: 'a',
-            cat1: 'b',
+                    data1: 30,
-            data1: 200,
+                    cat2: 'b',
-            cat2: 'a',
+                    data2: 400
-            data2: 60
+                }, {
-        }, {
+                    cat1: 'b',
-            cat1: 'c',
+                    data1: 200,
-            data1: 100,
+                    cat2: 'a',
-            cat2: 'c',
+                    data2: 60
-            data2: 200
+                }, {
-        }, {
+                    cat1: 'c',
-            cat1: 'd',
+                    data1: 100,
-            data1: 400,
+                    cat2: 'c',
-            cat2: 'd',
+                    data2: 200
-            data2: 800
+                }, {
-        }, {
+                    cat1: 'd',
-            cat2: 'e',
+                    data1: 400,
-            data2: 10
+                    cat2: 'd',
-        }, {
+                    data2: 800
-            cat2: 'f',
+                }, {
-            data2: 10
+                    cat2: 'e',
-        }]);
+                    data2: 10
+                }, {
+                    cat2: 'f',
+                    data2: 10
+                }]
+            });
+        });
+
+        it('throws when the column data contains undefined', () => {
+            expect(() => $$.convertColumnsToData([
+                ["cat1", "a", "b", "c", "d"],
+                ["data1", undefined]
+            ])).toThrowError(Error, /Source data is missing a component/);
+        });
     });
 
-    it('throws when the column data contains undefined', () => {
+    describe('$$.convertRowsToData', () => {
-        expect(() => $$.convertColumnsToData([
+        it('converts the row data to normalized data', () => {
-            ["cat1", "a", "b", "c", "d"],
+            const data = $$.convertRowsToData([
-            ["data1", undefined]
+                ['data1', 'data2', 'data3'],
-        ])).toThrowError(Error, /Source data is missing a component/);
+                [90, 120, 300],
+                [40, 160, 240],
+                [50, 200, 290],
+                [120, 160, 230],
+                [80, 130, 300],
+                [90, 220, 320]
+            ]);
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: 90,
+                    data2: 120,
+                    data3: 300
+                }, {
+                    data1: 40,
+                    data2: 160,
+                    data3: 240
+                }, {
+                    data1: 50,
+                    data2: 200,
+                    data3: 290
+                }, {
+                    data1: 120,
+                    data2: 160,
+                    data3: 230
+                }, {
+                    data1: 80,
+                    data2: 130,
+                    data3: 300
+                }, {
+                    data1: 90,
+                    data2: 220,
+                    data3: 320
+                }]
+            });
+        });
+
+        it('throws when the row data contains undefined', () => {
+            expect(() => $$.convertRowsToData([
+                ['data1', 'data2', 'data3'],
+                [40, 160, 240],
+                [90, 120, undefined]
+            ])).toThrowError(Error, /Source data is missing a component/);
+        });
     });
-});
 
-describe('$$.convertRowsToData', () => {
+    describe('$$.convertCsvToData', () => {
-    it('converts the row data to normalized data', () => {
+        it('converts the csv data to normalized data', () => {
-        const data = $$.convertRowsToData([
+            const data = $$.convertCsvToData(`data1,data2,data3
-            ['data1', 'data2', 'data3'],
+90,120,300
-            [90, 120, 300],
+40,160,240
-            [40, 160, 240],
+50,200,290
-            [50, 200, 290],
+120,160,230
-            [120, 160, 230],
+80,130,300
-            [80, 130, 300],
+90,220,320`);
-            [90, 220, 320]
+
-        ]);
+            expect(data).toEqual({
-
+                keys: ['data1', 'data2', 'data3'],
-        expect(data).toEqual([{
+                rows: [{
-            data1: 90,
+                    data1: '90',
-            data2: 120,
+                    data2: '120',
-            data3: 300
+                    data3: '300'
-        }, {
+                }, {
-            data1: 40,
+                    data1: '40',
-            data2: 160,
+                    data2: '160',
-            data3: 240
+                    data3: '240'
-        }, {
+                }, {
-            data1: 50,
+                    data1: '50',
-            data2: 200,
+                    data2: '200',
-            data3: 290
+                    data3: '290'
-        }, {
+                }, {
-            data1: 120,
+                    data1: '120',
-            data2: 160,
+                    data2: '160',
-            data3: 230
+                    data3: '230'
-        }, {
+                }, {
-            data1: 80,
+                    data1: '80',
-            data2: 130,
+                    data2: '130',
-            data3: 300
+                    data3: '300'
-        }, {
+                }, {
-            data1: 90,
+                    data1: '90',
-            data2: 220,
+                    data2: '220',
-            data3: 320
+                    data3: '320'
-        }]);
+                }]
+            });
+        });
+
+        it('converts one lined CSV data', () => {
+            const data = $$.convertCsvToData(`data1,data2,data3`);
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: null,
+                    data2: null,
+                    data3: null
+                }]
+            });
+        });
     });
 
-    it('throws when the row data contains undefined', () => {
+    describe('$$.convertTsvToData', () => {
-        expect(() => $$.convertRowsToData([
+        it('converts the tsv data to normalized data', () => {
-            ['data1', 'data2', 'data3'],
+            const data = $$.convertTsvToData(`data1\tdata2\tdata3
-            [40, 160, 240],
+90\t120\t300
-            [90, 120, undefined]
+40\t160\t240
-        ])).toThrowError(Error, /Source data is missing a component/);
+50\t200\t290
+120\t160\t230
+80\t130\t300
+90\t220\t320`);
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: '90',
+                    data2: '120',
+                    data3: '300'
+                }, {
+                    data1: '40',
+                    data2: '160',
+                    data3: '240'
+                }, {
+                    data1: '50',
+                    data2: '200',
+                    data3: '290'
+                }, {
+                    data1: '120',
+                    data2: '160',
+                    data3: '230'
+                }, {
+                    data1: '80',
+                    data2: '130',
+                    data3: '300'
+                }, {
+                    data1: '90',
+                    data2: '220',
+                    data3: '320'
+                }]
+            });
+        });
+
+        it('converts one lined TSV data', () => {
+            const data = $$.convertTsvToData(`data1\tdata2\tdata3`);
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: null,
+                    data2: null,
+                    data3: null
+                }]
+            });
+        });
+    });
+
+    describe('$$.convertDataToTargets', () => {
+
+        beforeEach(() => {
+            $$.cache = {};
+
+            $$.data = {
+                xs: []
+            };
+
+            $$.config = {
+                data_idConverter: (v) => v
+            };
+        });
+
+        it('converts the legacy data format into targets', () => {
+            const targets = $$.convertDataToTargets([ {
+                data1: 90,
+                data2: 120,
+                data3: 300
+            }, {
+                data1: 40,
+                data2: 160,
+                data3: 240
+            } ]);
+
+            expect(targets).toEqual([{
+                id: 'data1',
+                id_org: 'data1',
+                values: [ { x: 0, value: 90, id: 'data1', index: 0 }, { x: 1, value: 40, id: 'data1', index: 1 } ]
+            }, {
+                id: 'data2',
+                id_org: 'data2',
+                values: [ { x: 0, value: 120, id: 'data2', index: 0 }, { x: 1, value: 160, id: 'data2', index: 1 } ]
+            }, {
+                id: 'data3',
+                id_org: 'data3',
+                values: [ { x: 0, value: 300, id: 'data3', index: 0 }, { x: 1, value: 240, id: 'data3', index: 1 } ]
+            }]);
+        });
+
+        it('converts the data into targets', () => {
+            const targets = $$.convertDataToTargets({
+                keys: [ 'data1', 'data2', 'data3' ],
+                rows: [ {
+                    data1: 90,
+                    data2: 120,
+                    data3: 300
+                }, {
+                    data1: 40,
+                    data2: 160,
+                    data3: 240
+                } ]
+            });
+
+            expect(targets).toEqual([{
+                id: 'data1',
+                id_org: 'data1',
+                values: [ { x: 0, value: 90, id: 'data1', index: 0 }, { x: 1, value: 40, id: 'data1', index: 1 } ]
+            }, {
+                id: 'data2',
+                id_org: 'data2',
+                values: [ { x: 0, value: 120, id: 'data2', index: 0 }, { x: 1, value: 160, id: 'data2', index: 1 } ]
+            }, {
+                id: 'data3',
+                id_org: 'data3',
+                values: [ { x: 0, value: 300, id: 'data3', index: 0 }, { x: 1, value: 240, id: 'data3', index: 1 } ]
+            }]);
+        });
+
+    });
+
+    describe('$$.convertJsonToData', () => {
+
+        it('converts JSON as object (no keys provided)', () => {
+            const data = $$.convertJsonToData({
+                data1: [ 90, 40, 50, 120, 80, 90 ],
+                data2: [ 120, 160, 200, 160, 130, 220 ],
+                data3: [ 300, 240, 290, 230, 300, 320 ]
+            });
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: 90,
+                    data2: 120,
+                    data3: 300
+                }, {
+                    data1: 40,
+                    data2: 160,
+                    data3: 240
+                }, {
+                    data1: 50,
+                    data2: 200,
+                    data3: 290
+                }, {
+                    data1: 120,
+                    data2: 160,
+                    data3: 230
+                }, {
+                    data1: 80,
+                    data2: 130,
+                    data3: 300
+                }, {
+                    data1: 90,
+                    data2: 220,
+                    data3: 320
+                }]
+            });
+        });
+
+        it('converts JSON as rows (keys provided)', () => {
+            const data = $$.convertJsonToData([{
+                data1: 90,
+                data2: 120,
+                data3: 300,
+                unused: 42
+            }, {
+                data1: 40,
+                data2: 160,
+                data3: 240,
+                unused: 42
+            }, {
+                data1: 50,
+                data2: 200,
+                data3: 290,
+                unused: 42
+            }, {
+                data1: 120,
+                data2: 160,
+                data3: 230,
+                unused: 42
+            }, {
+                data1: 80,
+                data2: 130,
+                data3: 300,
+                unused: 42
+            }, {
+                data1: 90,
+                data2: 220,
+                data3: 320,
+                unused: 42
+            }], {
+                value: [ 'data1', 'data2', 'data3' ]
+            });
+
+            expect(data).toEqual({
+                keys: ['data1', 'data2', 'data3'],
+                rows: [{
+                    data1: 90,
+                    data2: 120,
+                    data3: 300
+                }, {
+                    data1: 40,
+                    data2: 160,
+                    data3: 240
+                }, {
+                    data1: 50,
+                    data2: 200,
+                    data3: 290
+                }, {
+                    data1: 120,
+                    data2: 160,
+                    data3: 230
+                }, {
+                    data1: 80,
+                    data2: 130,
+                    data3: 300
+                }, {
+                    data1: 90,
+                    data2: 220,
+                    data3: 320
+                }]
+            });
+        });
+
     });
 });

src/data.convert.js

@@ -1,5 +1,5 @@
 import { c3_chart_internal_fn } from './core';
-import { isValue, isUndefined, isDefined, notEmpty } from './util';
+import { isValue, isUndefined, isDefined, notEmpty, isArray } from './util';
 
 c3_chart_internal_fn.convertUrlToData = function (url, mimeType, headers, keys, done) {
     var $$ = this, type = mimeType ? mimeType : 'csv';
@@ -26,22 +26,20 @@ c3_chart_internal_fn.convertUrlToData = function (url, mimeType, headers, keys,
     });
 };
 c3_chart_internal_fn.convertXsvToData = function (xsv, parser) {
-    var rows = parser(xsv), d;
+    var [ keys, ...rows ] = parser.parseRows(xsv);
-    if (rows.length === 1) {
+    if (rows.length === 0) {
-        d = [{}];
+        return { keys, rows: [ keys.reduce((row, key) => Object.assign(row, { [key]: null }), {}) ] };
-        rows[0].forEach(function (id) {
-            d[0][id] = null;
-        });
     } else {
-        d = parser(xsv);
+        // [].concat() is to convert result into a plain array otherwise
+        // test is not happy because rows have properties.
+        return { keys, rows: [].concat(parser.parse(xsv)) };
     }
-    return d;
 };
 c3_chart_internal_fn.convertCsvToData = function (csv) {
-    return this.convertXsvToData(csv, this.d3.csvParse);
+    return this.convertXsvToData(csv, { parse: this.d3.csvParse, parseRows: this.d3.csvParseRows });
 };
 c3_chart_internal_fn.convertTsvToData = function (tsv) {
-    return this.convertXsvToData(tsv, this.d3.tsvParse);
+    return this.convertXsvToData(tsv, { parse: this.d3.tsvParse, parseRows: this.d3.tsvParseRows });
 };
 c3_chart_internal_fn.convertJsonToData = function (json, keys) {
     var $$ = this,
@@ -93,7 +91,7 @@ c3_chart_internal_fn.findValueInJson = function (object, path) {
 /**
  * Converts the rows to normalized data.
  * @param {any[][]} rows The row data
- * @return {Object[]}
+ * @return {Object}
  */
 c3_chart_internal_fn.convertRowsToData = (rows) => {
     const newRows = [];
@@ -109,16 +107,17 @@ c3_chart_internal_fn.convertRowsToData = (rows) => {
         }
         newRows.push(newRow);
     }
-    return newRows;
+    return { keys, rows: newRows };
 };
 
 /**
  * Converts the columns to normalized data.
  * @param {any[][]} columns The column data
- * @return {Object[]}
+ * @return {Object}
  */
 c3_chart_internal_fn.convertColumnsToData = (columns) => {
     const newRows = [];
+    const keys = [];
 
     for (let i = 0; i < columns.length; i++) {
         const key = columns[i][0];
@@ -131,16 +130,33 @@ c3_chart_internal_fn.convertColumnsToData = (columns) => {
             }
             newRows[j - 1][key] = columns[i][j];
         }
+        keys.push(key);
     }
 
-    return newRows;
+    return { keys, rows: newRows };
 };
 
+/**
+ * Converts the data format into the target format.
+ * @param {!Object} data
+ * @param {!Array} data.keys Ordered list of target IDs.
+ * @param {!Array} data.rows Rows of data to convert.
+ * @param {boolean} appendXs True to append to $$.data.xs, False to replace.
+ * @return {!Array}
+ */
 c3_chart_internal_fn.convertDataToTargets = function (data, appendXs) {
-    var $$ = this, config = $$.config,
+    var $$ = this, config = $$.config, targets, ids, xs, keys;
-        ids = $$.d3.keys(data[0]).filter($$.isNotX, $$),
+
-        xs = $$.d3.keys(data[0]).filter($$.isX, $$),
+    // handles format where keys are not orderly provided
-        targets;
+    if (isArray(data)) {
+        keys = Object.keys(data[ 0 ]);
+    } else {
+        keys = data.keys;
+        data = data.rows;
+    }
+
+    ids = keys.filter($$.isNotX, $$);
+    xs = keys.filter($$.isX, $$);
 
     // save x for update data by load when custom x and c3.x API
     ids.forEach(function (id) {