// SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project import test from 'tape'; import { createDataContainer, createIndexedDataContainer, compactArrowTable, ArrowDataContainer } from '@kepler.gl/utils'; import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants'; import * as arrow from 'apache-arrow'; function orderedNumericTable() { const cols = [ arrow.vectorFromArray([37, null, 38], new arrow.Float64()), arrow.vectorFromArray([-122, -123, -124], new arrow.Float32()), arrow.vectorFromArray([1000, null, 2000], new arrow.Int32()) ]; const fields = cols.map( (col, i) => new arrow.Field(['lat', 'lng', '2020'][i], col.type, true, new Map([['source', String(i)]])) ); const schema = new arrow.Schema(fields, new Map([['source', 'ordered']])); const batch = new arrow.RecordBatch( schema, arrow.makeData({type: new arrow.Struct(fields), length: 3, children: cols.map(c => c.data[0])}) ); return new arrow.Table(schema, [batch]); } test('ArrowDataContainer -> numeric headers preserve vector order through updates and IPC', t => { const source = orderedNumericTable(); const cols = source.schema.fields.map((_, i) => source.getChildAt(i)); const dc = new ArrowDataContainer({cols, fields: source.schema.fields}); const rows = [ [37, -122, 1000], [null, -123, null], [38, -124, 2000] ]; t.deepEqual(dc.flattenData(), rows, 'numeric names do not reorder values'); dc.update([ cols[0].slice(0, 1).concat(cols[0].slice(1)), cols[1], cols[2].slice(0, 2).concat(cols[2].slice(2)) ]); t.equal(dc.numChunks(), 3, 'staggered chunks remain aligned'); t.deepEqual(dc.flattenData(), rows); t.deepEqual( dc.getTable().schema.fields.map(f => f.name), ['lat', 'lng', '2020'] ); cols.forEach((col, i) => { t.equal(dc.getColumn(i).type.toString(), col.type.toString(), 'physical types stay aligned'); dc.getColumn(i).data.forEach(chunk => { t.equal(chunk.values.buffer, col.data[0].values.buffer, 'values are shared'); t.equal(chunk.nullBitmap.buffer, col.data[0].nullBitmap.buffer, 'validity is shared'); }); }); const restored = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); t.equal(restored.getChild('lat').get(0), 37); t.equal(restored.getChild('2020').get(0), 1000); dc.update(cols.map(col => col.slice(0, 0))); t.equal(dc.numRows(), 0); t.equal(dc.numColumns(), 3); t.deepEqual( dc.getTable().schema.fields.map(f => f.name), ['lat', 'lng', '2020'] ); t.equal(dc.getColumn(2).type.toString(), 'Int32', 'empty columns keep their types'); t.end(); }); test('compactArrowTable -> numeric headers keep values aligned with the schema', t => { const source = orderedNumericTable(); const compacted = compactArrowTable(source.concat(source), 1); t.equal(compacted.batches.length, 1); t.deepEqual( compacted.schema.fields.map(f => f.name), ['lat', 'lng', '2020'] ); source.schema.fields.forEach((field, i) => { t.deepEqual(Array.from(compacted.getChildAt(i)), [ ...source.getChildAt(i), ...source.getChildAt(i) ]); t.equal(compacted.getChildAt(i).type.toString(), field.type.toString()); t.deepEqual(compacted.schema.fields[i].metadata, field.metadata); }); t.deepEqual(compacted.schema.metadata, source.schema.metadata); t.equal(arrow.tableFromIPC(arrow.tableToIPC(compacted)).getChild('2020').get(0), 1000); t.equal(source.getChild('lat').get(0), 37, 'source values remain untouched'); t.end(); }); const data = [ [10, 20], // 0 [30, 40], // 1 [50, 60], // 2 [80, 90], // 3 [100, 110], // 4 [120, 130] // 5 ]; const indices = [1, 3, 5]; test('RowDataContainer', t => { const dc = createDataContainer(data); t.deepEqual(dc.numRows(), 6, `RowDataContainer should have expected number of rows`); t.deepEqual(dc.numColumns(), 2, `RowDataContainer should have expected number of columns`); t.deepEqual(dc.valueAt(2, 1), 60, `RowDataContainer.valueAt should return expected value`); t.deepEqual(dc.row(2).valueAt(1), 60, `RowDataContainer.row should return expected value`); t.deepEqual( dc.rowAsArray(2), [50, 60], `RowDataContainer.rowAsArray should return expected value` ); t.deepEqual(dc.flattenData(), data, `RowDataContainer.flattenData should return expected data`); t.deepEqual( dc.getPlainIndex(), [0, 1, 2, 3, 4, 5], `RowDataContainer.getPlainIndex should return expected indices` ); t.deepEqual( dc.map(row => row.valueAt(1)), [20, 40, 60, 90, 110, 130], `RowDataContainer.map should return expected array` ); t.deepEqual( dc.mapIndex(d => d), [{index: 0}, {index: 1}, {index: 2}, {index: 3}, {index: 4}, {index: 5}], `RowDataContainer.mapIndex should return expected array` ); t.deepEqual( dc.find(row => { return row.valueAt(1) === 60; }), dc.row(2), `RowDataContainer.find should return expected row` ); t.deepEqual( dc.reduce((acc, row) => { return acc + row.valueAt(1); }, 10), 460, `RowDataContainer.reduce should return expected value` ); t.end(); }); test('RowDataContainer.update replaces rows', t => { const dc = createDataContainer(data); dc.update([ [1, 2], [3, 4] ]); t.equal(dc.numRows(), 2, 'should replace with the new row count'); t.equal(dc.numColumns(), 2, 'should keep column count'); t.equal(dc.valueAt(0, 0), 1, 'should read values from the new rows'); t.deepEqual(dc.getPlainIndex(), [0, 1], 'should rebuild the index'); t.end(); }); test('RowDataContainer.append accepts a large batch without copying existing rows', t => { const first = [0, 10]; const rows = [first]; const dc = createDataContainer(rows); const incoming = Array.from({length: 200001}, (_, i) => [i + 1, i]); t.doesNotThrow(() => t.ok(dc.append(incoming), 'accepts all rows'), 'no argument limit'); t.equal(dc.flattenData(), rows, 'retains the backing array'); t.equal(dc.rowAsArray(0), first, 'retains existing row references'); t.equal(dc.numRows(), 200002, 'includes every appended row'); t.equal(dc.rowAsArray(200001), incoming[200000], 'retains the last incoming row'); t.end(); }); test('RowDataContainer.append and remove', t => { const dc = createDataContainer([ [10, 20], [30, 40], [50, 60] ]); t.equal( dc.append([ [70, 80], [90, 100] ]), true, 'should append matching rows' ); t.equal(dc.numRows(), 5, 'should grow without replacing existing rows'); t.equal(dc.valueAt(0, 0), 10, 'should keep the original first row'); t.equal(dc.valueAt(4, 1), 100, 'should read the last appended row'); t.deepEqual(dc.getPlainIndex(), [0, 1, 2, 3, 4], 'should expose contiguous indexes'); t.equal(dc.append([[1]]), false, 'should reject a wrong column count'); t.equal(dc.append([]), false, 'should reject an empty batch'); t.equal(dc.append(null), false, 'should reject a missing batch'); t.equal(dc.numRows(), 5, 'should leave rows unchanged after a rejected append'); t.equal(dc.replace(0, [11, 22]), true, 'should overwrite a row in place'); t.deepEqual(dc.rowAsArray(0), [11, 22], 'should read the replaced row'); t.equal(dc.replace(99, [1, 2]), false, 'should reject a replace with an out-of-range index'); t.equal(dc.replace(0, [1]), false, 'should reject a replace with the wrong column count'); t.deepEqual( dc.rowAsArray(0), [11, 22], 'should leave the row unchanged after a rejected replace' ); t.equal(dc.remove([1, 1, 3]), true, 'should drop unique valid indexes in one rebuild'); t.deepEqual( dc.flattenData(), [ [11, 22], [50, 60], [90, 100] ], 'should keep the remaining rows in order' ); t.deepEqual(dc.getPlainIndex(), [0, 1, 2], 'should rebuild indexes after remove'); t.equal(dc.remove([99]), false, 'should reject an out-of-range index'); t.equal(dc.remove([-1]), false, 'should reject a negative index'); t.equal(dc.numRows(), 3, 'should leave rows unchanged after a rejected remove'); dc.update([[1, 2]]); t.equal(dc.numRows(), 1, 'update should still replace after append/remove'); t.equal(dc.valueAt(0, 0), 1, 'update should write the new snapshot'); const empty = createDataContainer([]); t.equal( empty.append([[3, 4]]), true, 'should set column count from the first row on an empty table' ); t.equal(empty.numRows(), 1, 'empty table should accept the first append'); t.equal(empty.numColumns(), 2, 'empty table should take width from the first row'); t.end(); }); test('IndexedDataContainer', t => { const dc = createIndexedDataContainer(createDataContainer(data), indices); t.deepEqual(dc.numRows(), 3, `IndexedDataContainer should have expected number of rows`); t.deepEqual(dc.numColumns(), 2, `IndexedDataContainer should have expected number of columns`); t.deepEqual(dc.valueAt(2, 1), 130, `IndexedDataContainer.valueAt should return expected value`); t.deepEqual(dc.row(2).valueAt(1), 130, `IndexedDataContainer.row should return expected value`); t.deepEqual( dc.rowAsArray(2), [120, 130], `IndexedDataContainer.rowAsArray should return expected value` ); t.deepEqual( dc.flattenData(), [data[indices[0]], data[indices[1]], data[indices[2]]], `IndexedDataContainer.flattenData should return expected data` ); t.deepEqual( dc.getPlainIndex(), [0, 1, 2], `IndexedDataContainer.getPlainIndex should return expected indices` ); t.deepEqual( dc.map(row => row.valueAt(1)), [40, 90, 130], `IndexedDataContainer.map should return expected array` ); t.deepEqual( dc.mapIndex(d => d), [{index: 1}, {index: 3}, {index: 5}], `IndexedDataContainer.mapIndex should return expected array` ); t.deepEqual( dc.find(row => { return row.valueAt(1) === 90; }), dc.row(1), `IndexedDataContainer.find should return expected row` ); t.deepEqual( dc.reduce((acc, row) => { return acc + row.valueAt(1); }, 10), 270, `RowDataContainer.reduce should return expected value` ); t.end(); }); test('IndexedDataContainer preserves reordered rows across iterators and nested views', t => { const parent = createDataContainer(data); const dc = createIndexedDataContainer(parent, [5, 0, 3, 0]); const expected = [data[5], data[0], data[3], data[0]]; t.deepEqual(dc.flattenData(), expected, 'flattening preserves selection order and duplicates'); t.deepEqual( Array.from(dc.rows(), row => row.values()), expected, 'row iterator remaps rows' ); t.deepEqual(Array.from(dc.column(1)), [130, 20, 90, 20], 'column iterator remaps values'); t.deepEqual(dc.getPlainIndex(), [0, 1, 2, 3], 'plain indexes address the view'); const nested = createIndexedDataContainer(dc, [2, 0]); t.deepEqual(nested.flattenData(), [data[3], data[5]], 'nested selection composes mappings'); t.deepEqual( nested.mapIndex(({index}, container) => [index, container.rowAsArray(index)]), [ [2, data[3]], [0, data[5]] ], 'mapIndex pairs each immediate-parent index with its matching container' ); t.deepEqual(parent.flattenData(), data, 'reading a selection leaves the parent unchanged'); t.end(); }); test('IndexedDataContainer range callbacks distinguish local and parent indexes', t => { const parent = createDataContainer(data); const dc = createIndexedDataContainer(parent, [5, 0, 3, 1]); for (const sharedRow of [false, true]) { t.deepEqual( dc.map((row, index) => [index, row.valueAt(1)], sharedRow, {start: 1, end: 3}), [ [1, 20], [2, 90] ], 'map uses local indexes and an exclusive end with or without a shared row' ); } t.deepEqual( dc.mapIndex(({index}, container) => [index, container.valueAt(index, 1)], {start: 1, end: 3}), [ [0, 20], [3, 90] ], 'mapIndex uses parent indexes with the same local range' ); t.deepEqual( dc.map((row, index) => [index, row.valueAt(1)], false, {start: 2, end: 99}), [ [2, 90], [3, 40] ], 'map caps the end at the view length' ); t.deepEqual( dc.mapIndex(({index}) => index, {start: 2, end: 99}), [3, 1], 'mapIndex caps the end at the view length' ); for (const range of [{start: 2, end: 2}, {start: 4}, {end: 0}]) { const unexpected = () => t.fail('empty ranges must not invoke callbacks'); t.deepEqual(dc.map(unexpected, false, range), [], 'empty map range'); t.deepEqual(dc.mapIndex(unexpected, range), [], 'empty mapIndex range'); } t.end(); }); test('IndexedDataContainer find and reduce use selection order and local indexes', t => { const dc = createIndexedDataContainer(createDataContainer(data), [5, 0, 3, 0]); for (const sharedRow of [false, true]) { const visited = []; const found = dc.find((row, index) => { visited.push(index); return row.valueAt(0) === 10; }, sharedRow); t.deepEqual(found.values(), data[0], 'find returns the first matching selected row'); t.deepEqual(visited, [0, 1], 'find stops after the first match and exposes local indexes'); t.equal( dc.find(row => row.valueAt(0) === -1, sharedRow), undefined, 'no match is undefined' ); t.deepEqual( dc.reduce((acc, row, index) => acc.concat([[index, row.valueAt(0)]]), [], sharedRow), [ [0, 120], [1, 10], [2, 80], [3, 10] ], 'reduce visits selected rows in order, including duplicate parent rows' ); } t.end(); }); test('IndexedDataContainer empty selection retains columns without reading parent rows', t => { const dc = createIndexedDataContainer(createDataContainer(data), []); const unexpected = () => t.fail('empty selection must not invoke callbacks'); const initial = {total: 0}; t.equal(dc.numRows(), 0, 'selection contains no rows'); t.equal(dc.numColumns(), 2, 'parent column count is retained'); t.deepEqual(dc.getPlainIndex(), [], 'no local indexes'); t.deepEqual(dc.flattenData(), [], 'no exported rows'); t.deepEqual(Array.from(dc.rows()), [], 'no iterated rows'); t.deepEqual(Array.from(dc.column(1)), [], 'no iterated values'); t.deepEqual(dc.map(unexpected), [], 'map is empty'); t.deepEqual(dc.mapIndex(unexpected), [], 'mapIndex is empty'); t.equal(dc.find(unexpected), undefined, 'find has no match'); t.equal(dc.reduce(unexpected, initial), initial, 'reduce returns the original initial value'); t.end(); }); test('ArrowDataContainer -> compactArrowTable collapses record batches over the limit', t => { const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); const combined = batchA.concat(batchB); t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); const belowLimit = compactArrowTable(combined, 255); t.equal( belowLimit.batches.length, combined.batches.length, 'compactArrowTable should leave tables at or under maxArrowBatches unchanged' ); combined.schema.metadata.set('geo', JSON.stringify({columns: {lng: {encoding: 'WKB'}}})); combined.schema.fields[0].metadata.set('ARROW:extension:name', 'geoarrow.wkb'); const compacted = compactArrowTable(combined, 1); t.equal(compacted.numRows, 2, 'compactArrowTable should keep all rows'); t.equal(compacted.batches.length, 1, 'compactArrowTable should collapse into one batch'); t.ok( compacted.schema.metadata.get('geo'), 'compactArrowTable should keep GeoParquet schema metadata used to create layers' ); t.equal( compacted.schema.fields[0].metadata.get('ARROW:extension:name'), 'geoarrow.wkb', 'compactArrowTable should keep geoarrow field metadata' ); const cols = Array.from({length: combined.numCols}, (_, i) => combined.getChildAt(i)).filter( Boolean ); const arrowDc = new ArrowDataContainer({ cols, fields: combined.schema.fields.map((field, fieldIdx) => ({ name: field.name, fieldIdx })), arrowTable: compactArrowTable(combined, 1) }); t.equal(arrowDc.numRows(), 2, 'ArrowDataContainer should keep all rows'); t.equal(arrowDc.numChunks(), 1, 'ArrowDataContainer should store a compacted table'); t.deepEqual(arrowDc.valueAt(0, 0), -122.4, 'compacted container should preserve values'); t.end(); }); test('ArrowDataContainer -> update does not collapse record batches', t => { const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); const colsA = Array.from({length: batchA.numCols}, (_, i) => batchA.getChildAt(i)).filter( Boolean ); const dc = new ArrowDataContainer({ cols: colsA, fields: batchA.schema.fields.map((field, fieldIdx) => ({ name: field.name, fieldIdx })), arrowTable: batchA }); const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); const combined = batchA.concat(batchB); dc.update(combined); t.equal(dc.numRows(), 2, 'update should keep all rows'); t.equal( dc.numChunks(), combined.batches.length, 'update should not compact batches during incremental loading' ); t.deepEqual(dc.valueAt(1, 0), -122.5, 'update should preserve newly appended values'); t.end(); }); test('ArrowDataContainer -> worker GeoArrow cache follows progressive chunks', t => { const batchA = arrow.tableFromJSON([{value: 1}]); const batchB = arrow.tableFromJSON([{value: 2}]); const featureTypes = {point: true, line: false, polygon: false}; const cachedA = { binaryGeometries: ['a'], bounds: [0, 0, 0, 0], featureTypes, meanCenters: [[0, 0]] }; const cachedB = { binaryGeometries: ['b'], bounds: [1, 1, 1, 1], featureTypes, meanCenters: [[1, 1]] }; const dc = new ArrowDataContainer({ cols: [batchA.getChildAt(0)], fields: [{name: 'value', fieldIdx: 0}], arrowTable: batchA, geoArrowBinaryData: {0: {0: cachedA}} }); const combined = batchA.concat(batchB); dc.update(combined, {geoArrowBinaryData: {0: {1: cachedB}}}); t.equal(dc.getGeoArrowBinaryData(0, 0), cachedA, 'should retain the first parsed chunk'); t.equal(dc.getGeoArrowBinaryData(0, 1), cachedB, 'should append the new parsed chunk'); t.deepEqual( dc.getGeoArrowBinaryData(0).binaryGeometries, ['a', 'b'], 'should expose all cached chunks in record-batch order' ); const compacted = compactArrowTable(combined, 1); const compactedCache = {...cachedA, binaryGeometries: ['all'], meanCenters: [[0.5, 0.5]]}; dc.update(compacted, {geoArrowBinaryData: {0: {0: compactedCache}}}); t.equal( dc.getGeoArrowBinaryData(0, 0), compactedCache, 'compaction should replace cache entries because chunk indexes reset' ); t.equal(dc.getGeoArrowBinaryData(0, 1), undefined, 'stale progressive chunks should be removed'); const largeMeanCenters = Array.from({length: 150000}, (_, index) => [index, index]); const largeCache = {...compactedCache, meanCenters: largeMeanCenters}; dc.update(compacted, {geoArrowBinaryData: {0: {0: largeCache}}}); let combinedCache; t.doesNotThrow(() => { combinedCache = dc.getGeoArrowBinaryData(0); }, 'large compacted centroid caches should not overflow the call stack'); t.equal( combinedCache.meanCenters.length, largeMeanCenters.length, 'large compacted centroid caches should keep every center' ); t.equal(dc.remove([0]), true, 'should remove a row from the cached table'); t.equal(dc.valueAt(0, 0), 2, 'should retain only the second row'); t.equal(dc.getGeoArrowBinaryData(0), undefined, 'row deletion should discard cached geometry'); dc.update(batchA, {geoArrowBinaryData: {0: {0: cachedA}}}); t.equal(dc.remove([0]), true, 'should remove the final cached row'); t.equal(dc.numRows(), 0, 'should leave an empty table'); t.equal(dc.getGeoArrowBinaryData(0), undefined, 'empty tables must not retain cached geometry'); t.end(); }); function finishArrowVector(builder) { const finished = builder.finish(); if (finished instanceof arrow.Vector) { return finished; } return finished.toVector(); } function tableFromColumnValues(name, type, values, metadata) { const builder = arrow.makeBuilder({type, nullValues: [null]}); values.forEach(value => builder.append(value)); const vector = finishArrowVector(builder); const field = new arrow.Field(name, type, true, metadata); return new arrow.Table(new arrow.Schema([field]), {[name]: vector}); } function concatColumnTables(name, type, leftValues, rightValues, metadata) { return tableFromColumnValues(name, type, leftValues, metadata).concat( tableFromColumnValues(name, type, rightValues, metadata) ); } function jsArrowCell(value) { if (value == null) { return null; } if (ArrayBuffer.isView(value)) { return Array.from(value); } if (typeof value.toArray === 'function' && value.numChildren === 0) { return Array.from(value.toArray()); } if (typeof value.get === 'function' && typeof value.length === 'number') { const out = []; for (let i = 0; i < value.length; i++) { const valid = typeof value.isValid === 'function' ? value.isValid(i) : true; out.push(valid ? jsArrowCell(value.get(i)) : null); } return out; } return value; } function columnCells(table) { const column = table.getChildAt(0); return Array.from({length: column.length}, (_, i) => { if (typeof column.isValid === 'function' && !column.isValid(i)) { return null; } return jsArrowCell(column.get(i)); }); } test('compactArrowTable -> concat does not merge chunks; Vector has no combineChunks', t => { const batchA = arrow.tableFromJSON([{lng: -122.4}]); const batchB = arrow.tableFromJSON([{lng: -122.5}]); const combined = batchA.concat(batchB); const column = combined.getChildAt(0); t.equal( typeof column.combineChunks, 'undefined', 'apache-arrow JS Vector has concat, not combineChunks' ); t.ok(column.data.length > 1, 'Table.concat keeps one chunk per record batch'); t.equal(typeof column.concat, 'function', 'Vector.concat exists but does not collapse chunks'); t.end(); }); test('compactArrowTable -> dictionary, WKB, and nested GeoArrow keep values', t => { const geoMeta = metadata => { const map = new Map(); map.set(GEOARROW_METADATA_KEY, metadata); return map; }; const dictType = new arrow.Dictionary(new arrow.Utf8(), new arrow.Int8()); const dictTable = concatColumnTables('city', dictType, ['sf', null], ['nyc']); const dictCompacted = compactArrowTable(dictTable, 1); t.equal(dictCompacted.batches.length, 1, 'dictionary column should collapse to one batch'); t.deepEqual( columnCells(dictCompacted), ['sf', null, 'nyc'], 'dictionary values and nulls should survive the Builder rebuild' ); const wkbPoint = (lng, lat) => { const bytes = new Uint8Array(21); const view = new DataView(bytes.buffer); view.setUint8(0, 1); view.setUint32(1, 1, true); view.setFloat64(5, lng, true); view.setFloat64(13, lat, true); return bytes; }; const wkbTable = concatColumnTables( 'geometry', new arrow.Binary(), [wkbPoint(-122.4, 37.8), null], [wkbPoint(-122.5, 37.9)], geoMeta(GEOARROW_EXTENSIONS.WKB) ); wkbTable.schema.metadata.set('geo', JSON.stringify({columns: {geometry: {encoding: 'WKB'}}})); const wkbCompacted = compactArrowTable(wkbTable, 1); t.equal(wkbCompacted.batches.length, 1, 'WKB binary column should collapse to one batch'); t.deepEqual( columnCells(wkbCompacted), columnCells(wkbTable), 'WKB bytes and nulls should survive the Builder rebuild' ); t.equal( wkbCompacted.schema.fields[0].metadata.get(GEOARROW_METADATA_KEY), GEOARROW_EXTENSIONS.WKB, 'geoarrow.wkb field metadata should be copied' ); t.ok(wkbCompacted.schema.metadata.get('geo'), 'GeoParquet schema metadata should be copied'); const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); const pointTable = concatColumnTables( 'geom', pointType, [[-122.4, 37.8], null], [[-122.5, 37.9]], geoMeta(GEOARROW_EXTENSIONS.POINT) ); const pointCompacted = compactArrowTable(pointTable, 1); t.equal(pointCompacted.batches.length, 1, 'geoarrow.point should collapse to one batch'); t.deepEqual( columnCells(pointCompacted), [[-122.4, 37.8], null, [-122.5, 37.9]], 'geoarrow.point coordinates must not become NaN after Builder rebuild' ); const lineType = new arrow.List(new arrow.Field('vertices', pointType, false)); const lineTable = concatColumnTables( 'geom', lineType, [ [ [-122.4, 37.8], [-122.5, 37.9] ] ], [ [ [0, 0], [1, 1] ] ], geoMeta(GEOARROW_EXTENSIONS.LINESTRING) ); const lineCompacted = compactArrowTable(lineTable, 1); t.equal(lineCompacted.batches.length, 1, 'geoarrow.linestring should collapse to one batch'); t.deepEqual( columnCells(lineCompacted), columnCells(lineTable), 'nested geoarrow.linestring coordinates should survive the Builder rebuild' ); const polygonType = new arrow.List(new arrow.Field('rings', lineType, false)); const ring = [ [0, 0], [1, 0], [1, 1], [0, 0] ]; const polygonTable = concatColumnTables( 'geom', polygonType, [[ring]], [[ring]], geoMeta(GEOARROW_EXTENSIONS.POLYGON) ); const polygonCompacted = compactArrowTable(polygonTable, 1); t.equal(polygonCompacted.batches.length, 1, 'geoarrow.polygon should collapse to one batch'); t.deepEqual( columnCells(polygonCompacted), columnCells(polygonTable), 'nested geoarrow.polygon coordinates should survive the Builder rebuild' ); t.end(); }); test('compactArrowTable -> duplicate field names keep both columns', t => { const duplicateNameTable = (floatValues, intValues) => { const schema = new arrow.Schema([ new arrow.Field('x', new arrow.Float64()), new arrow.Field('x', new arrow.Int32()) ]); const floats = arrow.vectorFromArray(floatValues, new arrow.Float64()); const ints = arrow.vectorFromArray(intValues, new arrow.Int32()); const data = arrow.makeData({ type: new arrow.Struct(schema.fields), children: [floats.data[0], ints.data[0]], length: floatValues.length }); return new arrow.Table(new arrow.RecordBatch(schema, data)); }; const combined = duplicateNameTable([1.5], [1]).concat(duplicateNameTable([2.5], [2])); t.equal(combined.numCols, 2, 'fixture should expose both duplicate-named columns'); t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); const compacted = compactArrowTable(combined, 1); t.equal(compacted.numCols, 2, 'compaction should not drop a duplicate-named column'); t.equal(compacted.numRows, 2, 'compaction should keep all rows'); t.equal( compacted.batches.length, 1, 'duplicate-named columns should still collapse to one batch' ); t.deepEqual( [compacted.getChildAt(0).get(0), compacted.getChildAt(0).get(1)], [1.5, 2.5], 'first duplicate-named column should keep its float values' ); t.deepEqual( [compacted.getChildAt(1).get(0), compacted.getChildAt(1).get(1)], [1, 2], 'second duplicate-named column should keep its int values' ); t.end(); }); test('compactArrowTable -> Builder failure keeps the original table', t => { const batchA = arrow.tableFromJSON([{lng: -122.4}]); const batchB = arrow.tableFromJSON([{lng: -122.5}]); const combined = batchA.concat(batchB); const exploding = { batches: combined.batches, numCols: combined.numCols, schema: combined.schema, getChildAt() { throw new Error('cannot compact'); } }; const result = compactArrowTable(exploding, 1); t.equal( result, exploding, 'compactArrowTable should return the original table when the Builder path throws' ); t.end(); }); function arrowContainerFromTable(table) { const cols = Array.from({length: table.numCols}, (_, i) => table.getChildAt(i)).filter(Boolean); return new ArrowDataContainer({ cols, fields: table.schema.fields.map((field, fieldIdx) => ({ name: field.name, fieldIdx })), arrowTable: table }); } test('ArrowDataContainer.append replace and remove', t => { const dc = arrowContainerFromTable( arrow.tableFromJSON([ {id: 'a', lat: 1}, {id: 'b', lat: 2} ]) ); t.equal( dc.append([ ['c', 3], ['d', 4] ]), true, 'should append matching primitive/dictionary rows' ); t.equal(dc.numRows(), 4, 'should grow without replacing existing rows'); t.equal(dc.valueAt(0, 0), 'a', 'should keep the original first id'); t.equal(dc.valueAt(3, 1), 4, 'should read the last appended lat'); t.equal(dc.numChunks(), 1, 'append should compact concat batches'); t.equal(dc.append([[1]]), false, 'should reject a wrong column count'); t.equal(dc.append([]), false, 'should reject an empty batch'); t.equal(dc.append(null), false, 'should reject a missing batch'); t.equal(dc.numRows(), 4, 'should leave rows unchanged after a rejected append'); t.equal(dc.replace(1, ['B', 22]), true, 'should overwrite a row by concat'); t.equal(dc.valueAt(1, 0), 'B', 'should read the replaced id'); t.equal(dc.valueAt(1, 1), 22, 'should read the replaced lat'); t.equal(dc.replace(99, ['z', 1]), false, 'should reject a replace with an out-of-range index'); t.equal(dc.replace(0, ['z']), false, 'should reject a replace with the wrong column count'); t.equal(dc.valueAt(0, 0), 'a', 'should leave the row unchanged after a rejected replace'); t.equal(dc.remove([1, 1, 3]), true, 'should drop unique valid indexes in one rebuild'); t.equal(dc.numRows(), 2, 'should keep the remaining rows'); t.equal(dc.valueAt(0, 0), 'a', 'should keep the former row 0'); t.equal(dc.valueAt(1, 0), 'c', 'should keep the former row 2'); t.deepEqual(dc.getPlainIndex(), [0, 1], 'should rebuild indexes after remove'); t.equal(dc.remove([99]), false, 'should reject an out-of-range index'); t.equal(dc.remove([-1]), false, 'should reject a negative index'); t.equal(dc.numRows(), 2, 'should leave rows unchanged after a rejected remove'); t.equal(dc.remove([0, 1]), true, 'should allow removing every remaining row'); t.equal(dc.numRows(), 0, 'should keep an empty table with the original schema'); t.equal( dc.append([['e', 5]]), true, 'empty table should accept an append that matches the schema' ); t.equal(dc.numRows(), 1, 'empty table should grow from the append'); t.equal(dc.valueAt(0, 0), 'e', 'appended id should survive an empty-table concat'); t.end(); }); test('ArrowDataContainer row edits preserve column positions and schema', t => { for (const names of [ ['2', '1'], ['x', 'x', 'x_1'], ['__proto__', 'value'] ]) { const original = names.map((_, i) => i + 1); const added = original.map(value => value * 10); const replaced = original.map(value => value * 100); const columns = original.map((value, i) => arrow.vectorFromArray([value], i % 2 ? new arrow.Float64() : new arrow.Int32()) ); const fields = names.map( (name, i) => new arrow.Field(name, columns[i].type, true, new Map([['index', String(i)]])) ); const schema = new arrow.Schema(fields, new Map([['source', 'row-edit']])); const batch = new arrow.RecordBatch( schema, arrow.makeData({ type: new arrow.Struct(fields), length: 1, children: columns.map(column => column.data[0]) }) ); fields.forEach((field, i) => { batch.schema.fields[i] = field; }); const dc = arrowContainerFromTable(new arrow.Table(schema, [batch])); t.ok(dc.append([added]), `append accepts ${names.join(', ')}`); t.deepEqual(dc.flattenData(), [original, added], 'append keeps column values aligned'); t.ok(dc.replace(0, replaced), 'replace succeeds'); t.deepEqual(dc.flattenData(), [replaced, added], 'replace keeps every column'); t.ok(dc.remove([0]), 'remove succeeds'); t.deepEqual(dc.flattenData(), [added], 'remove keeps the remaining values'); t.deepEqual(dc.getTable().schema, schema, 'names, types and metadata survive row edits'); } t.end(); }); test('ArrowDataContainer.append rejects nested and geoarrow columns', t => { const originalWarn = console.warn; console.warn = () => {}; try { const listTable = new arrow.Table({ vals: arrow.vectorFromArray([ [1, 2], [3, 4] ]) }); const listDc = arrowContainerFromTable(listTable); t.equal(listDc.append([[[5, 6]]]), false, 'should reject a List column'); t.equal(listDc.numRows(), 2, 'rejected append must not mutate a List table'); t.equal(listDc.remove([0]), true, 'remove should still concat slices of nested columns'); t.equal(listDc.numRows(), 1, 'should drop the nested row by slice'); const geoMeta = new Map(); geoMeta.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB); const wkbTable = tableFromColumnValues( 'geometry', new arrow.Binary(), [new Uint8Array([1, 2, 3])], geoMeta ); const geoDc = arrowContainerFromTable(wkbTable); t.equal(geoDc.append([[new Uint8Array([4, 5, 6])]]), false, 'should reject a geoarrow column'); t.equal(geoDc.numRows(), 1, 'rejected append must not mutate a geoarrow table'); } finally { console.warn = originalWarn; } t.end(); }); test('Arrow row edits preserve retained raw timestamps through compaction', t => { for (const [type, values] of [ [ new arrow.TimestampMicrosecond('UTC'), [9007199254740993n, 9007199254740995n, 9007199254740997n] ], [ new arrow.TimestampNanosecond('UTC'), [1700000000000000123n, 1700000000000000456n, 1700000000000000789n] ] ]) { const timestamp = arrow.makeVector({ type, data: new BigInt64Array([0n, ...values]), nullBitmap: new Uint8Array([0b1011]), nullCount: 1 }); const source = new arrow.Table({ id: arrow.vectorFromArray(['skip', 'a', 'b', 'c'], new arrow.Utf8()), time: timestamp }).slice(1); const expected = [values[0], null, values[2]]; const raw = table => { const column = table.getChildAt(1); const storage = column.toArray(); return Array.from({length: column.length}, (_, i) => (column.isValid(i) ? storage[i] : null)); }; for (const [name, mutate, retained] of [ ['append', dc => dc.append([['d', 0]]), [...expected, 0n]], ['delete middle', dc => dc.remove([1]), [values[0], values[2]]] ]) { const dc = arrowContainerFromTable(source); t.ok(mutate(dc), `${type} ${name} succeeds`); t.deepEqual(raw(dc.getTable()), retained, `${type} ${name} retains exact raw values`); t.deepEqual( raw(arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()))), retained, 'IPC retains raw storage' ); t.deepEqual(raw(source), expected, 'original snapshot is unchanged'); t.equal(dc.numChunks(), 1, 'row edits respect the compaction limit'); } } t.end(); }); test('Arrow row edits preserve shared dictionary serialization', t => { const column = arrow.vectorFromArray( ['a', null, 'b'], new arrow.Dictionary(new arrow.Utf8(), new arrow.Int8(), 17) ); const fields = ['left', '2020'].map( name => new arrow.Field(name, column.type, true, new Map([['source', name]])) ); const schema = new arrow.Schema(fields, new Map([['source', 'shared-dictionary']])); const source = new arrow.Table(schema, {0: column, 1: column}); const dc = arrowContainerFromTable(source); t.ok(dc.append([['c', 'd']]), 'append succeeds with separate incoming dictionary values'); t.deepEqual( dc.flattenData(), [ ['a', 'a'], [null, null], ['b', 'b'], ['c', 'd'] ], 'values stay aligned' ); const assertIPC = expected => { let restored; t.doesNotThrow(() => { restored = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); }, 'edited table is serializable'); if (restored) { t.deepEqual( restored.schema.fields.map((_, i) => Array.from(restored.getChildAt(i))), expected, 'IPC preserves every dictionary value' ); } t.equal( dc.getTable().schema.metadata.get('source'), 'shared-dictionary', 'table metadata survives' ); t.deepEqual( dc.getTable().schema.fields.map(field => field.metadata.get('source')), ['left', '2020'], 'field metadata survives' ); }; assertIPC([ ['a', null, 'b', 'c'], ['a', null, 'b', 'd'] ]); t.ok(dc.remove([0, 1, 2]), 'deletion succeeds'); assertIPC([['c'], ['d']]); t.ok(dc.remove([0]), 'delete all succeeds'); t.ok(dc.append([['e', 'f']]), 'refill succeeds'); assertIPC([['e'], ['f']]); t.deepEqual(Array.from(source.getChildAt(0)), ['a', null, 'b'], 'source values stay untouched'); t.doesNotThrow(() => arrow.tableToIPC(source), 'source remains serializable'); t.end(); }); test('Arrow row edits reject dictionary index overflow atomically', t => { for (const [Index, capacity] of [ [arrow.Int8, 128], [arrow.Uint8, 256] ]) { const type = new arrow.Dictionary(new arrow.Utf8(), new Index()); for (const empty of [false, true]) { const column = arrow.vectorFromArray(empty ? [] : ['original'], type); const source = new arrow.Table({value: column}); const dc = arrowContainerFromTable(source); const snapshot = dc.getTable(); const rows = Array.from({length: capacity + 1}, (_, i) => [`value-${i}`]); t.notOk(dc.append(rows), 'oversized incoming dictionary is rejected before indexes wrap'); t.equal(dc.getTable(), snapshot, 'rejection retains the snapshot'); t.deepEqual(dc.flattenData(), empty ? [] : [['original']], 'rejection changes no values'); } const initial = Array.from({length: capacity}, (_, i) => `value-${i}`); const dc = arrowContainerFromTable( new arrow.Table({value: arrow.vectorFromArray(initial, type)}) ); const snapshot = dc.getTable(); t.notOk(dc.append([['overflow']]), 'combined dictionary overflow is rejected'); t.equal(dc.getTable(), snapshot, 'combined overflow is atomic'); } t.end(); }); test('Arrow deletion preserves nested timestamp storage without decoding it', t => { const values = [1700000000000000123n, 1700000000000000456n, 1700000000000000789n]; const time = arrow.makeVector({ type: new arrow.TimestampNanosecond(), data: new BigInt64Array(values) }); const type = new arrow.Struct([new arrow.Field('time', time.type, true)]); const nested = arrow.makeVector(arrow.makeData({type, length: 3, children: time.data})); const source = new arrow.Table({nested}); const dc = arrowContainerFromTable(source); t.ok(dc.remove([1]), 'nested deletion succeeds'); const raw = table => Array.from(new arrow.Vector(table.getChildAt(0).data.map(data => data.children[0])).toArray()); t.deepEqual(raw(dc.getTable()), [values[0], values[2]], 'nested raw values are exact'); t.deepEqual( raw(arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()))), [values[0], values[2]], 'nested IPC is exact' ); t.deepEqual(raw(source), values, 'original nested buffers are unchanged'); t.end(); }); test('Arrow shared timestamp dictionaries preserve raw values and IPC after edits', t => { const type = new arrow.Dictionary(new arrow.TimestampNanosecond('UTC'), new arrow.Int8(), 17); const timestamp = arrow.makeVector({ type: type.dictionary, data: new BigInt64Array([1700000000000000123n, 1700000000000000456n]) }); const data = arrow.makeData({type, data: new Int8Array([0, 1]), dictionary: timestamp}); const column = arrow.makeVector(data); const source = new arrow.Table( new arrow.Schema(['left', 'right'].map(name => new arrow.Field(name, type, true))), {0: column, 1: column} ); const dc = arrowContainerFromTable(source); t.ok(dc.append([[0, 1000]]), 'shared timestamp append succeeds'); const ipc = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); for (let i = 0; i < 2; i++) { const col = ipc.getChildAt(i); const raw = col.data.flatMap(chunk => Array.from( chunk.values.slice(0, chunk.length), index => chunk.dictionary.data[0].values[index] ) ); t.deepEqual( raw, [1700000000000000123n, 1700000000000000456n, i ? 1000000000n : 0n], 'shared dictionary keeps exact raw timestamp entries' ); } t.equal( dc.getColumn(0).data[0].dictionary, dc.getColumn(1).data[0].dictionary, 'output columns share the same dictionary' ); t.doesNotThrow(() => arrow.tableToIPC(source), 'source dictionary stays serializable'); t.end(); }); test('Arrow shared dictionaries retain differing physical index widths', t => { const dictionary = arrow.vectorFromArray(['a', 'b'], new arrow.Utf8()); const columns = [new arrow.Int8(), new arrow.Int16()].map(indices => arrow.makeVector({ type: new arrow.Dictionary(dictionary.type, indices, 17), data: new indices.ArrayType([0, 1]), dictionary }) ); const source = new arrow.Table({narrow: columns[0], wide: columns[1]}); const dc = arrowContainerFromTable(source); t.ok(dc.append([['c', 'd']]), 'mixed-width append succeeds'); t.ok(dc.remove([0]), 'mixed-width deletion succeeds'); const ipc = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); t.deepEqual(Array.from(ipc.getChildAt(0)), ['b', 'c'], 'narrow values survive IPC'); t.deepEqual(Array.from(ipc.getChildAt(1)), ['b', 'd'], 'wide values survive IPC'); t.deepEqual( ipc.schema.fields.map(field => field.type.indices.bitWidth), [8, 16], 'physical index widths are preserved' ); t.equal( dc.getColumn(0).data[0].dictionary, dc.getColumn(1).data[0].dictionary, 'both columns retain one shared dictionary' ); t.deepEqual(Array.from(source.getChildAt(0)), ['a', 'b'], 'source stays unchanged'); t.end(); }); test('Arrow deletion retains timestamp storage inside nested dictionaries', t => { const time = new arrow.TimestampNanosecond('UTC'); const type = new arrow.List(new arrow.Field('item', time)); const values = [1234567890123456789n, 1234567890123456791n]; const dictionary = arrow.makeVector({ type, valueOffsets: new Int32Array([0, 1, 2]), child: arrow.makeData({type: time, data: new BigInt64Array(values)}) }); const column = arrow.makeVector({ type: new arrow.Dictionary(type, new arrow.Int8(), 17), data: new Int8Array([0, 1, 0, 1]), dictionary }); const source = new arrow.Table({nested: column}); const dc = arrowContainerFromTable(source); t.ok(dc.remove([1]), 'nested dictionary deletion succeeds'); const ipc = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); t.deepEqual( Array.from(ipc.getChildAt(0), value => value.toArray()[0]), [values[0], values[0], values[1]], 'nested dictionary IPC keeps exact timestamp values' ); t.deepEqual( Array.from(source.getChildAt(0), value => value.toArray()[0]), [values[0], values[1], values[0], values[1]], 'source nested dictionary is unchanged' ); t.end(); }); test('Arrow deletion reconciles nested dictionary mappings across progressive batches', t => { const batch = values => { const dictionary = arrow.vectorFromArray(values, new arrow.Utf8()); const type = new arrow.Dictionary(dictionary.type, new arrow.Int8(), 17); const column = indexes => arrow.makeVector({ type: new arrow.List(new arrow.Field('item', type, true)), valueOffsets: new Int32Array([0, 1, 2, 3]), child: arrow.makeData({type, data: new Int8Array(indexes), dictionary}) }); return new arrow.Table({left: column([0, 1, 0]), right: column([1, 0, 1])}); }; const first = batch(['a', 'b']); const source = first.concat(batch(['b', 'a'])); const before = arrow.tableToIPC(source); const dc = arrowContainerFromTable(first); dc.update(source); t.ok(dc.remove([1, 4]), 'deletion accepts different dictionaries in progressive batches'); const rows = table => Array.from({length: table.numRows}, (_, index) => table.schema.fields.map((_, column) => Array.from(table.getChildAt(column).get(index))) ); const expected = [ [['a'], ['b']], [['a'], ['b']], [['b'], ['a']], [['b'], ['a']] ]; t.deepEqual(rows(dc.getTable()), expected, 'retained indexes use the unified dictionary'); t.equal(dc.numChunks(), 1, 'edited table has one batch'); t.deepEqual( rows(arrow.tableFromIPC(arrow.tableToIPC(dc.getTable()))), expected, 'shared dictionary survives IPC despite equivalent distinct source type objects' ); t.deepEqual(arrow.tableToIPC(source), before, 'progressive source buffers remain unchanged'); t.end(); }); test('compactArrowTable -> nested timestamp fallback keeps exact source storage', t => { const type = new arrow.TimestampNanosecond(); const nested = new arrow.List(new arrow.Field('time', type, true)); const raw = [1700000000000000123n, 1700000000000000456n]; const batches = raw.map( value => new arrow.Table({ times: arrow.makeVector({ type: nested, valueOffsets: new Int32Array([0, 1]), child: arrow.makeData({type, data: new BigInt64Array([value])}) }) }) ); const source = batches[0].concat(batches[1]); const compacted = compactArrowTable(source, 1); t.deepEqual( compacted.getChildAt(0).data.flatMap(data => Array.from(data.children[0].values)), raw, 'ordinary import compaction must not round nested timestamps through milliseconds' ); t.end(); }); test('Arrow deletion counts only retained nested dictionary entries toward capacity', t => { const batch = prefix => { const dictionary = arrow.vectorFromArray( Array.from({length: 128}, (_, i) => `${prefix}${i}`), new arrow.Utf8() ); const type = new arrow.Dictionary(dictionary.type, new arrow.Int8(), 17); const column = arrow.makeVector({ type: new arrow.List(new arrow.Field('item', type, true)), valueOffsets: Int32Array.from({length: 129}, (_, i) => i), child: arrow.makeData({ type, data: Int8Array.from({length: 128}, (_, i) => i), dictionary }) }); return new arrow.Table({values: column}); }; const first = batch('a'); const source = first.concat(batch('b')); const original = arrow.tableToIPC(source); const dc = arrowContainerFromTable(first); dc.update(source); const indexes = Array.from({length: 256}, (_, i) => i).filter(i => i !== 0 && i !== 128); t.ok(dc.remove(indexes), 'two retained dictionary values fit Int8 despite deleted entries'); const restored = arrow.tableFromIPC(arrow.tableToIPC(dc.getTable())); t.deepEqual( Array.from(restored.getChildAt(0), value => Array.from(value)), [['a0'], ['b0']], 'only the retained values survive IPC' ); t.deepEqual(arrow.tableToIPC(source), original, 'source dictionaries are unchanged'); t.end(); });