// SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project import test from 'tape'; import sinon from 'sinon'; import {WorkerFarm} from '@loaders.gl/worker-utils'; import {convertGeoArrowToBinaryFeatureCollection} from '@loaders.gl/gis'; import {KeplerTable} from '@kepler.gl/table'; import GeoJsonLayer from '../../../src/layers/src/geojson-layer/geojson-layer'; import { isKeplerGlMap, makeProgressIterator, filesToDataPayload, fileCacheToProgressiveSteps, processFileData, parseGeoArrowBatchesOnWorkers, remoteDatasetFromUrl, processArrowBatches, getGeoJsonFromLoaderResult, readBatch } from '@kepler.gl/processors'; import { GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, getDatasetRefreshIntervalMs } from '@kepler.gl/constants'; import * as arrow from 'apache-arrow'; // The standalone bundle has distinct constructors, like DuckDB's bundled Arrow. import * as foreignArrow from 'apache-arrow/Arrow.es2015.min'; import {convertArrowToSchema} from '@loaders.gl/schema-utils'; import {parsedFields, parsedRows} from 'test/fixtures/row-object'; import { savedStateV1InteractionCoordinate as keplerglMap, parsedFields as parsedKeplerMapFields } from 'test/fixtures/state-saved-v1-7'; test('#file-handler -> isKeplerGlMap', t => { t.equal( isKeplerGlMap('{datasets: [], info: {app: "kepler.gl"}, config: {}}'), false, 'Should return false when passing a json string' ); t.equal( isKeplerGlMap({datasets: [], info: {app: 'kepler.gl'}, config: {}}), true, 'Should return true when object is a kepler map' ); t.equal( isKeplerGlMap({datasets: [], info: {app: 'kepler.gl'}}), false, 'Should return false when object is not a kepler map' ); t.end(); }); test('#file-handler -> makeProgressIterator', async t => { // mock AsyncIterator returned by loarder.gl patchInBatches // Ideally should run this in browser-headless async function* mock() { let bytesUsed = 0; let value = 0; let b = 0; await new Promise(resolve => setTimeout(resolve, 100)); while (b < 2) { b += 1; bytesUsed += 10; value += 1; yield { data: [{value}], bytesUsed }; } } const asyncIterator = mock(); const info = {size: 100}; const progress = makeProgressIterator(asyncIterator, info); const batch1 = await progress.next(); const batch2 = await progress.next(); const batch3 = await progress.next(); const expected1 = { value: { data: [{value: 1}], bytesUsed: 10, progress: {rowCount: 1, rowCountInBatch: 1, percent: 0.1} }, done: false }; const expected2 = { value: { data: [{value: 2}], bytesUsed: 20, progress: {rowCount: 2, rowCountInBatch: 1, percent: 0.2} }, done: false }; const expected3 = { value: undefined, done: true }; t.deepEqual(batch1, expected1, 'batch1 should be correct'); t.deepEqual(batch2, expected2, 'batch2 should be correct'); t.deepEqual(batch3, expected3, 'batch3 should be correct'); t.end(); }); test('#file-handler -> filesToDataPayload', t => { const fileCache = [ { data: { fields: parsedFields, rows: parsedRows }, info: {label: 'rows-data.json', format: 'row'} }, { data: { datasets: [ { data: { fields: parsedKeplerMapFields, rows: keplerglMap.datasets[0].data.allData }, info: {id: 'a5ybmwl2d', label: 'geojson_as_string_small.csv', color: [53, 92, 125]} } ], config: keplerglMap.config }, info: {label: 'keplergl-map.json', format: 'keplergl'} } ]; const result = filesToDataPayload(fileCache); // const expectedResults = [ // { // datasets: [{data, info}], // config: { // version: 'v1', // config: {} // }, // options: {centerMap: true} // }, // {datasets: [{data, info}]} // ]; t.equal(result.length, 2, 'result shoud have 2 entries'); t.deepEqual( Object.keys(result[0]), ['datasets', 'config', 'options'], 'result[0] should have 3 keys' ); t.equal(result[0].datasets, fileCache[1].data.datasets, 'should save keplergl map datasets'); t.equal(result[0].config, fileCache[1].data.config, 'should save keplergl map config'); t.deepEqual( result[0].options, {centerMap: true}, 'should save keplergl map set {centerMap: true}' ); t.deepEqual(Object.keys(result[1]), ['datasets'], 'result[0] should have 1 key'); t.deepEqual( result[1].datasets[0].data, fileCache[0].data, 'should pass file data to datasets only' ); t.deepEqual( Object.keys(result[1].datasets[0].info), ['id', 'label', 'format'], 'result[0] datasets[0].info should have 3 key' ); t.end(); }); test('#file-handler -> filesToDataPayload autoCreateLayers', t => { const fileCache = [ { data: { fields: parsedFields, rows: parsedRows }, info: {label: 'rows-data.json', format: 'row'} }, { data: { datasets: [], config: keplerglMap.config }, info: {label: 'keplergl-map.json', format: 'keplergl'} } ]; const result = filesToDataPayload(fileCache, {autoCreateLayers: false}); t.deepEqual( result[0].options, {centerMap: true, autoCreateLayers: false}, 'should keep kepler map centerMap and forward autoCreateLayers' ); t.deepEqual( result[1].options, {autoCreateLayers: false}, 'should forward autoCreateLayers for dataset files' ); t.end(); }); test('#file-handler -> remoteDatasetFromUrl waits for the remote download', t => { const dataset = remoteDatasetFromUrl('https://example.com/quakes.csv?sv=1', 'csv'); t.equal(dataset.info.type, 'externally-hosted', 'should be a remote dataset'); t.equal(dataset.info.label, 'quakes.csv', 'should name the card from the url'); t.deepEqual(dataset.data, {fields: [], rows: []}, 'should not include downloaded rows'); t.equal(dataset.metadata.source, 'https://example.com/quakes.csv?sv=1'); t.equal(dataset.metadata.sourceFormat, 'csv', 'should keep the selected format'); const payload = filesToDataPayload([dataset]); t.equal(payload[0].datasets[0].info.type, 'externally-hosted', 'Add Data keeps the remote type'); t.deepEqual(payload[0].datasets[0].data, {fields: [], rows: []}, 'Add Data still has no rows'); t.end(); }); test('#file-handler -> filesToDataPayload remote metadata', t => { const fileCache = [ { data: { fields: parsedFields, rows: parsedRows }, info: { id: 'remote-ds', label: 'quakes.csv', format: 'row', type: 'externally-hosted' }, metadata: { source: 'https://example.com/quakes.csv', sourceFormat: 'csv' } } ]; const result = filesToDataPayload(fileCache); t.equal(result.length, 1, 'result should have 1 entry'); t.equal(result[0].datasets[0].info.type, 'externally-hosted', 'should pass type'); t.deepEqual( result[0].datasets[0].metadata, {source: 'https://example.com/quakes.csv', sourceFormat: 'csv'}, 'should pass remote source metadata' ); t.end(); }); test('#file-handler -> processFileData one-shot load does not enable polling', async t => { const rows = [{lat: 1, lng: 2}]; const local = await processFileData({ content: {fileName: 'local.csv', data: rows}, fileCache: [] }); t.equal(local[0].info.type, undefined, 'local files are not marked externally-hosted'); t.equal(local[0].metadata, undefined, 'local files get no remote refresh metadata'); const sized = await processFileData({ content: {fileName: 'local.csv', data: rows, fileSize: 50 * 1024 * 1024}, fileCache: [] }); t.equal( sized[0].metadata.size, 50 * 1024 * 1024, 'local files keep their byte size for a later warning' ); const remote = await processFileData({ content: { fileName: 'remote.csv', data: rows, sourceUrl: 'https://example.com/remote.csv' }, fileCache: [] }); t.equal(remote[0].info.type, 'externally-hosted', 'URL loads are still externally-hosted'); t.equal( remote[0].metadata.refreshIntervalMs, undefined, 'a one-shot URL load does not set a poll interval' ); t.equal( getDatasetRefreshIntervalMs(remote[0].metadata), 0, 'polling helper treats omitted interval as off' ); t.end(); }); test('#file-handler -> processFileData persists remote file format', async t => { const cache = await processFileData({ content: { fileName: 'quakes.csv', data: [{lat: 1, lng: 2}], sourceUrl: 'https://example.com/abc123?sv=1', keplerFormat: 'csv' }, fileCache: [] }); t.equal(cache[0].info.type, 'externally-hosted', 'should mark the dataset as externally-hosted'); t.equal(cache[0].info.format, 'row', 'processor format stays row for CSV'); t.equal(cache[0].metadata.source, 'https://example.com/abc123?sv=1'); t.equal( cache[0].metadata.sourceFormat, 'csv', 'should persist the file format, not the processor format' ); t.equal( typeof cache[0].metadata.lastFetchedAt, 'number', 'should record lastFetchedAt on remote load' ); t.equal(cache[0].metadata.etag, undefined, 'should omit etag when the fetch did not provide one'); t.equal( cache[0].metadata.lastModified, undefined, 'should omit lastModified when the fetch did not provide one' ); const withValidators = await processFileData({ content: { fileName: 'quakes.csv', data: [{lat: 1, lng: 2}], sourceUrl: 'https://example.com/quakes.csv', keplerFormat: 'csv', etag: '"abc"', lastModified: 'Wed, 21 Oct 2015 07:28:00 GMT' }, fileCache: [] }); t.equal(withValidators[0].metadata.etag, '"abc"', 'should persist ETag for the next refresh'); t.equal( withValidators[0].metadata.lastModified, 'Wed, 21 Oct 2015 07:28:00 GMT', 'should persist Last-Modified for the next refresh' ); const parquetCache = await processFileData({ content: { fileName: 'data.parquet', data: [{lat: 1, lng: 2}], sourceUrl: 'https://example.com/data.parquet' }, fileCache: [] }); t.equal( parquetCache[0].metadata.sourceFormat, 'parquet', 'should infer parquet from the filename when no format was selected' ); t.end(); }); test('#file-handler -> processFileData remote ids do not collide on filename', async t => { const rows = [{lat: 1, lng: 2}]; const local = await processFileData({ content: {fileName: 'quakes.csv', data: rows}, fileCache: [] }); const remoteA = await processFileData({ content: { fileName: 'quakes.csv', data: rows, sourceUrl: 'https://a.example.com/quakes.csv' }, fileCache: [] }); const remoteB = await processFileData({ content: { fileName: 'quakes.csv', data: rows, sourceUrl: 'https://b.example.com/quakes.csv' }, fileCache: [] }); const remoteAAgain = await processFileData({ content: { fileName: 'quakes.csv', data: rows, sourceUrl: 'https://a.example.com/quakes.csv' }, fileCache: [] }); t.equal(local[0].info.label, 'quakes.csv', 'local label stays the filename'); t.equal(remoteA[0].info.label, 'quakes.csv', 'remote label stays the filename'); t.notEqual( local[0].info.id, remoteA[0].info.id, 'local and remote files with the same name get different ids' ); t.notEqual( remoteA[0].info.id, remoteB[0].info.id, 'two remote URLs with the same filename get different ids' ); t.equal( remoteA[0].info.id, remoteAAgain[0].info.id, 'reloading the same URL keeps a stable id so progressive batches can update in place' ); t.ok(!String(remoteA[0].info.id).includes('http'), 'id is a hash, not the raw URL'); t.end(); }); test('#file-handler -> convertArrowToSchema uses a single apache-arrow copy', t => { // Parquet loading serializes the Arrow schema via loaders.gl. If kepler.gl // and @loaders.gl/schema-utils resolve different apache-arrow packages, that // switch-on-constructor check throws `arrow type not supported: ` // (minified to e.g. `tL` in exported HTML). const table = new arrow.Table({ lat: arrow.vectorFromArray(new Float64Array([37.8])), lng: arrow.vectorFromArray(new Float64Array([-122.4])), name: arrow.vectorFromArray(['alpha'], new arrow.Utf8()) }); let schema; t.doesNotThrow(() => { schema = convertArrowToSchema(table.schema); }, 'schema conversion should not fail with a constructor identity mismatch'); t.deepEqual( schema.fields.map(field => field.type), ['float64', 'float64', 'utf8'], 'should serialize primitive Arrow types from the shared apache-arrow copy' ); t.end(); }); test('#file-handler -> processArrowBatches skip compact for incremental loads', t => { const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); const combined = batchA.concat(batchB); t.ok(combined.batches.length > 1, 'fixture should have multiple record batches'); const skipped = processArrowBatches(combined.batches, {compact: false}); t.equal( skipped.cols[0].data.length, combined.batches.length, 'compact:false should leave record batches unchanged' ); const compacted = processArrowBatches(combined.batches); t.equal(compacted.cols[0].data.length, 1, 'default processArrowBatches should compact'); t.equal(compacted.cols[0].length, 2, 'compacted table should keep all rows'); t.end(); }); test('#file-handler -> parseGeoArrowBatchesOnWorkers parses the latest chunk', async t => { const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); const batchA = new arrow.Table({ geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType) }); const batchB = new arrow.Table({ geometry: arrow.vectorFromArray( [ [-122.5, 37.9], [-122.6, 38] ], pointType ) }); batchA.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); batchB.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); const combined = batchA.concat(batchB); const processed = processArrowBatches(combined.batches, {compact: false}); const calls = []; const binaryDataFromGeoArrow = { binaryGeometries: [{shape: 'binary-feature-collection'}], bounds: [-122.6, 37.9, -122.5, 38], featureTypes: {point: true, line: false, polygon: false}, meanCenters: [ [-122.5, 37.9], [-122.6, 38] ] }; const parsed = await parseGeoArrowBatchesOnWorkers(processed, { latestOnly: true, parseOnWorker: async input => { calls.push(input); return {chunkIndex: input.chunkIndex, binaryDataFromGeoArrow}; } }); t.equal(calls.length, 1, 'should schedule one worker for the newest chunk'); t.equal(calls[0].chunkIndex, 1, 'should parse the second record batch'); t.equal(calls[0].chunkOffset, 1, 'should offset global feature ids by earlier row counts'); t.equal( calls[0].geometryEncoding, GEOARROW_EXTENSIONS.POINT, 'should preserve the native GeoArrow encoding' ); t.equal(parsed[0][1], binaryDataFromGeoArrow, 'should cache worker output by column and chunk'); t.end(); }); test('#file-handler -> separated points reach workers and fallback with all coordinates', async t => { const type = new arrow.Struct( ['x', 'y', 'z'].map(name => new arrow.Field(name, new arrow.Float64(), false)) ); const batches = [ [ {x: 10, y: 20, z: 5}, {x: 30, y: 40, z: 15} ], [{x: 50, y: 60, z: 25}] ].map(points => { const table = new arrow.Table({geometry: arrow.vectorFromArray(points, type)}); table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); return table; }); const table = arrow.tableFromIPC(arrow.tableToIPC(batches[0].concat(batches[1]), 'file')); const bytes = arrow.tableToIPC(table, 'file'); const processed = processArrowBatches(table.batches, {compact: false}); for (const failWorker of [false, true]) { const result = await parseGeoArrowBatchesOnWorkers(processed, { parseOnWorker: async input => { t.equal(input.chunkData.type.listSize, 3, 'worker input has XYZ coordinate size'); if (failWorker) throw new Error('worker unavailable'); return parseGeoArrowWorkerInput(input); } }); const {layer} = await layerFromFile({ info: {id: 'separated'}, data: {...processed, geoArrowBinaryData: result} }); t.deepEqual( layer.dataToFeature.flatMap(data => Array.from(data.points.positions.value)), [10, 20, 5, 30, 40, 15, 50, 60, 25], 'worker/cache and fallback retain all axes' ); t.deepEqual( layer.meta.bounds, [10, 20, 50, 60], 'cached bounds include latitude from every batch' ); t.deepEqual( layer.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value)), [0, 1, 2], 'cached feature ids retain unequal batch offsets' ); } t.deepEqual(arrow.tableToIPC(table, 'file'), bytes, 'worker preparation preserves source data'); t.end(); }); test('#file-handler -> worker fallback accepts points and lines from another Arrow runtime', async t => { const pointType = new foreignArrow.FixedSizeList( 2, new foreignArrow.Field('xy', new foreignArrow.Float64()) ); for (const encoding of [GEOARROW_EXTENSIONS.POINT, GEOARROW_EXTENSIONS.LINESTRING]) { const isPoint = encoding === GEOARROW_EXTENSIONS.POINT; const chunks = [ [ [10, 20], [30, 40] ], [[50, 60]] ].map(points => { // Exact child/offset lengths keep unrelated IPC padding out of line fixtures. const coordinates = isPoint ? points : points.flatMap(point => [point, point]); const child = foreignArrow .vectorFromArray(coordinates, pointType) .data[0].slice(0, coordinates.length); return isPoint ? child : foreignArrow.makeData({ type: new foreignArrow.List(new foreignArrow.Field('vertices', pointType)), length: points.length, valueOffsets: Int32Array.from(points.map((_, i) => i * 2).concat(points.length * 2)), child }); }); const geoColumn = foreignArrow.makeVector(chunks); const field = { name: 'geometry', type: 'geoarrow', fieldIdx: 0, metadata: new Map([[GEOARROW_METADATA_KEY, encoding]]) }; const table = new foreignArrow.Table({geometry: geoColumn}); const originalBytes = foreignArrow.tableToIPC(table); t.notOk(chunks[0] instanceof arrow.Data, 'fixture uses a separate Arrow runtime'); for (const latestOnly of [false, true]) { const result = await parseGeoArrowBatchesOnWorkers( {fields: [field], cols: [geoColumn]}, { latestOnly, parseOnWorker: async () => { throw new Error('worker unavailable'); } } ); t.deepEqual(Object.keys(result[0]), latestOnly ? ['1'] : ['0', '1'], 'selected batches'); for (const [chunkIndex, binary] of Object.entries(result[0])) { const last = chunkIndex === '1'; const points = last ? [[50, 60]] : [ [10, 20], [30, 40] ]; const rowIds = last ? [2] : [0, 1]; const geometry = binary.binaryGeometries[0][isPoint ? 'points' : 'lines']; t.deepEqual( Array.from(geometry.positions.value), (isPoint ? points : points.flatMap(point => [point, point])).flat(), 'fallback preserves all coordinates' ); t.deepEqual( Array.from(geometry.globalFeatureIds.value), isPoint ? rowIds : rowIds.flatMap(id => [id, id]), 'fallback preserves global row ids' ); t.deepEqual(binary.bounds, last ? [50, 60, 50, 60] : [10, 20, 30, 40], 'bounds'); t.deepEqual(binary.meanCenters, points, 'centroids remain aligned with rows'); } } t.deepEqual(foreignArrow.tableToIPC(table), originalBytes, 'fallback preserves source data'); } t.end(); }); test('#file-handler -> parseGeoArrowBatchesOnWorkers combines compacted worker chunks', async t => { const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); const batchA = new arrow.Table({ geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType) }); const batchB = new arrow.Table({ geometry: arrow.vectorFromArray([[-122.5, 37.9]], pointType) }); batchA.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); batchB.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); const processed = processArrowBatches(batchA.concat(batchB).batches, {compact: false}); const calls = []; const parsed = await parseGeoArrowBatchesOnWorkers(processed, { targetChunkCount: 1, parseOnWorker: async input => { calls.push(input); const x = input.chunkIndex === 0 ? -122.4 : -122.5; const y = input.chunkIndex === 0 ? 37.8 : 37.9; return { chunkIndex: input.chunkIndex, binaryDataFromGeoArrow: { binaryGeometries: [{chunkIndex: input.chunkIndex}], bounds: [x, y, x, y], featureTypes: {point: true, line: false, polygon: false}, meanCenters: [[x, y]] } }; } }); t.equal(calls.length, 2, 'should schedule one worker job per original record batch'); t.deepEqual(Object.keys(parsed[0]), ['0'], 'should expose one cache chunk after compaction'); t.deepEqual( parsed[0][0].binaryGeometries, [{chunkIndex: 0}, {chunkIndex: 1}], 'should retain worker geometry in record-batch order' ); t.deepEqual(parsed[0][0].bounds, [-122.5, 37.8, -122.4, 37.9], 'should merge bounds'); t.deepEqual( parsed[0][0].meanCenters, [ [-122.4, 37.8], [-122.5, 37.9] ], 'should merge mean centers' ); t.end(); }); test('#file-handler -> compacted worker cache handles large centroid arrays', async t => { const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); const table = new arrow.Table({ geometry: arrow.vectorFromArray([[-122.4, 37.8]], pointType) }); table.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); const processed = processArrowBatches(table.batches, {compact: false}); const largeMeanCenters = Array.from({length: 150000}, () => [-122.4, 37.8]); const parsed = await parseGeoArrowBatchesOnWorkers(processed, { targetChunkCount: 1, parseOnWorker: async input => ({ chunkIndex: input.chunkIndex, binaryDataFromGeoArrow: { binaryGeometries: [], bounds: [-122.4, 37.8, -122.4, 37.8], featureTypes: {point: true, line: false, polygon: false}, meanCenters: largeMeanCenters } }) }); t.equal( parsed[0][0].meanCenters.length, largeMeanCenters.length, 'should combine every centroid without spreading them into function arguments' ); t.end(); }); function geoArrowPointBatches(pointBatches = [[[-122.4, 37.8]], [[-122.5, 37.9]]]) { const pointType = new arrow.FixedSizeList(2, new arrow.Field('xy', new arrow.Float64(), false)); let rowIndex = 0; return pointBatches.map(points => { const geometry = arrow.makeData({ type: pointType, length: points.length, child: arrow.makeData({type: new arrow.Float64(), data: Float64Array.from(points.flat())}) }); const row = arrow.makeData({ type: new arrow.Int32(), data: Int32Array.from(points.map(() => rowIndex++)) }); const schema = new arrow.Schema([ new arrow.Field('geometry', pointType), new arrow.Field('row', row.type) ]); schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); return new arrow.RecordBatch( schema, arrow.makeData({ type: new arrow.Struct(schema.fields), length: points.length, children: [geometry, row] }) ); }); } // Replace only worker transport; successful jobs use the same converter as the // loaders.gl worker. File processing, table import and layer cache reads stay real. async function withGeoArrowWorker(parse, run) { const originalWorker = globalThis.Worker; globalThis.Worker = class {}; const stub = sinon.stub(WorkerFarm, 'getWorkerFarm').callsFake(() => ({ getWorkerPool: () => ({ startJob: async () => { let resolve; let reject; const result = new Promise((yes, no) => { resolve = yes; reject = no; }); return { result, postMessage: (_type, {input}) => { Promise.resolve() .then(() => parse(input)) .then(value => resolve({result: value}), reject); } }; } }) })); try { await run(); } finally { stub.restore(); if (originalWorker === undefined) { delete globalThis.Worker; } else { globalThis.Worker = originalWorker; } } } function parseGeoArrowWorkerInput(input) { const {chunkData, chunkIndex, chunkOffset, geometryEncoding} = input; const vector = arrow.makeVector( new arrow.Data( chunkData.type, chunkData.offset, chunkData.length, chunkData.nullCount, chunkData.buffers, chunkData.children, chunkData.dictionary ) ); return { chunkIndex, binaryDataFromGeoArrow: convertGeoArrowToBinaryFeatureCollection(vector, geometryEncoding, { chunkIndex: 0, chunkOffset, calculateMeanCenters: true, triangle: true }) }; } async function layerFromFile(file) { const dataset = new KeplerTable({info: file.info, color: [0, 0, 0]}); await dataset.importData({data: file.data}); const layer = new GeoJsonLayer({dataId: dataset.id}); layer.config.columns.geojson = {value: 'geometry', fieldIdx: 0}; layer.updateLayerMeta(dataset); return {dataset, layer}; } async function assertCompleteGeoArrowLayer( t, file, points = [ [-122.4, 37.8], [-122.5, 37.9] ] ) { const {dataset, layer} = await layerFromFile(file); t.equal(dataset.dataContainer.numRows(), points.length, 'table keeps every row'); t.deepEqual(layer.centroids, points, 'centroids retain row order'); t.deepEqual( layer.meta.bounds, [ Math.min(...points.map(point => point[0])), Math.min(...points.map(point => point[1])), Math.max(...points.map(point => point[0])), Math.max(...points.map(point => point[1])) ], 'bounds include every batch' ); const featureIds = layer.dataToFeature.flatMap(data => Array.from(data.points.globalFeatureIds.value) ); t.deepEqual( featureIds, points.map((_, index) => index), 'geometry retains every feature row id' ); t.deepEqual( featureIds.map(index => dataset.dataContainer.valueAt(index, 1)), featureIds, 'geometry maps to the matching attribute row' ); return {dataset, layer}; } for (const failure of ['first', 'last', 'all', 'missing']) { test(`#file-handler -> ${failure} worker failure preserves the complete compacted layer`, async t => { await withGeoArrowWorker( input => { if (failure === 'missing' && input.chunkIndex === 1) { return {chunkIndex: input.chunkIndex, binaryDataFromGeoArrow: null}; } if (failure === 'all' || input.chunkIndex === (failure === 'first' ? 0 : 1)) { throw new Error('worker failed'); } return parseGeoArrowWorkerInput(input); }, async () => { const [file] = await processFileData({ content: { fileName: 'points.arrow', data: geoArrowPointBatches(), prepareProgressiveReplay: true }, fileCache: [] }); const {dataset} = await assertCompleteGeoArrowLayer(t, file); t.equal(dataset.dataContainer.numChunks(), 1, 'final table is compacted'); await assertCompleteGeoArrowLayer(t, {...file, data: file.progressiveData[0]}, [ [-122.4, 37.8] ]); } ); t.end(); }); } function deferred() { let resolve; let reject; const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); return {promise, resolve, reject}; } test('#file-handler -> out-of-order workers retain the successful fast path', async t => { const jobs = [deferred(), deferred()]; const started = deferred(); const outputs = []; await withGeoArrowWorker( input => { outputs[input.chunkIndex] = parseGeoArrowWorkerInput(input); if (outputs.filter(Boolean).length === 2) started.resolve(); return jobs[input.chunkIndex].promise; }, async () => { const pending = processFileData({ content: {fileName: 'points.arrow', data: geoArrowPointBatches()}, fileCache: [] }); await started.promise; jobs[1].resolve(outputs[1]); jobs[0].resolve(outputs[0]); const [file] = await pending; const {layer} = await assertCompleteGeoArrowLayer(t, file); t.equal( layer.dataToFeature[0], outputs[0].binaryDataFromGeoArrow.binaryGeometries[0], 'first batch reuses the worker geometry' ); t.equal( layer.dataToFeature[1], outputs[1].binaryDataFromGeoArrow.binaryGeometries[0], 'second batch reuses the worker geometry' ); } ); t.end(); }); for (const failEmpty of [false, true]) { test(`#file-handler -> ${failEmpty ? 'failed' : 'successful'} empty worker batch`, async t => { const outputs = []; await withGeoArrowWorker( input => { if (failEmpty && input.chunkData.length === 0) throw new Error('empty worker failed'); const result = parseGeoArrowWorkerInput(input); outputs[input.chunkIndex] = result; return result; }, async () => { const [file] = await processFileData({ content: { fileName: 'points.arrow', data: geoArrowPointBatches([[[-122.4, 37.8]], [], [[-122.5, 37.9]]]) }, fileCache: [] }); const {layer} = await assertCompleteGeoArrowLayer(t, file); t.equal( layer.dataToFeature[0] === outputs[0].binaryDataFromGeoArrow.binaryGeometries[0], !failEmpty, 'only a failed empty batch triggers complete fallback' ); } ); t.end(); }); } test('#file-handler -> failed latest worker batch preserves incremental row ids', async t => { await withGeoArrowWorker( () => Promise.reject(new Error('worker failed')), async () => { const [file] = await processFileData({ content: {fileName: 'points.arrow', data: geoArrowPointBatches(), skipArrowCompact: true}, fileCache: [] }); const latest = file.data.geoArrowBinaryData[0][1]; t.deepEqual(latest.meanCenters, [[-122.5, 37.9]], 'latest batch has its centroid'); t.deepEqual( Array.from(latest.binaryGeometries[0].points.globalFeatureIds.value), [1], 'fallback keeps the original global row offset' ); } ); t.end(); }); test('#file-handler -> geometry errors still reject after worker fallback', async t => { const malformed = arrow.tableFromArrays({geometry: new Int32Array([1])}); malformed.schema.fields[0].metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT); await withGeoArrowWorker( () => Promise.reject(new Error('worker failed')), async () => { try { await processFileData({ content: {fileName: 'invalid.arrow', data: malformed.batches}, fileCache: [] }); t.fail('invalid geometry must reject instead of returning a partial cache'); } catch (error) { t.ok(error instanceof Error, 'synchronous geometry parse errors propagate'); } } ); t.end(); }); test('#file-handler -> decode errors after a valid batch still reject', async t => { const error = new Error('Arrow decode failed'); async function* batches() { yield {data: geoArrowPointBatches()}; throw error; } const iterator = readBatch(batches(), 'broken.arrow'); await iterator.next(); try { await iterator.next(); t.fail('a decode error must not turn into a completed partial file'); } catch (caught) { t.equal(caught, error, 'the original decode error propagates'); } t.end(); }); test('#file-handler -> processArrowBatches Int64/Uint64 columns', async t => { const table = arrow.tableFromArrays({ hexId: new BigUint64Array([610625465232654335n, 610625465081659391n]), total: new BigInt64Array([29436887n, 40685227n]) }); let result; t.doesNotThrow(() => { result = processArrowBatches(table.batches); }, 'should process Arrow Int64/Uint64 columns without BigInt TypeError'); t.equal(result.fields[0].name, 'hexId'); t.equal(result.fields[0].type, 'integer', 'uint64 columns should stay integer, not h3'); t.equal(result.fields[1].name, 'total'); t.equal(result.fields[1].type, 'integer', 'int64 counts should stay integer'); const processed = await processFileData({ content: {fileName: 'congo.parquet', data: table.batches}, fileCache: [] }); t.equal(processed.length, 1, 'processFileData should accept uint64 Arrow batches'); t.equal(processed[0].data.fields[0].type, 'integer'); t.end(); }); test('#file-handler -> processFileData Feature array is geojson', async t => { const features = [ { type: 'Feature', properties: {name: 'alpha'}, geometry: {type: 'Point', coordinates: [-122.4, 37.8]} }, { type: 'Feature', properties: {name: 'beta'}, geometry: {type: 'Point', coordinates: [-122.5, 37.9]} } ]; const processed = await processFileData({ content: {fileName: 'places.geojsonl', data: features}, fileCache: [] }); t.equal( processed[0].info.format, 'geojson', 'an array of Features should not be treated as rows' ); t.equal(processed[0].data.rows.length, 2, 'should keep both features'); t.end(); }); test('#file-handler -> shapefile-shaped loader output becomes GeoJSON', async t => { const data = { data: [ { type: 'Feature', properties: {name: 'alpha'}, geometry: {type: 'Point', coordinates: [-122.4, 37.8]} } ] }; t.equal( getGeoJsonFromLoaderResult(data).features.length, 1, 'should unwrap shapefile Feature arrays' ); const processed = await processFileData({ content: {fileName: 'places.shp', data}, fileCache: [] }); t.equal(processed[0].info.format, 'geojson', 'shapefile output should process as geojson'); t.equal(processed[0].data.rows.length, 1, 'should keep the shapefile feature'); t.end(); }); async function readLastAggregatedBatch(iterator, fileName) { const generator = readBatch(iterator, fileName); let last; for await (const batch of generator) { last = batch; } return last; } test('#file-handler -> empty GIS loader output becomes an empty GeoJSON dataset', async t => { const emptyShapefile = await processFileData({ content: {fileName: 'empty.shp', data: {data: []}}, fileCache: [] }); t.equal(emptyShapefile[0].info.format, 'geojson', 'empty shapefile output should be geojson'); t.equal(emptyShapefile[0].data.rows.length, 0, 'should keep zero features'); const emptyTable = await processFileData({ content: { fileName: 'empty.fgb', data: {shape: 'geojson-table', type: 'FeatureCollection', features: []} }, fileCache: [] }); t.equal(emptyTable[0].info.format, 'geojson', 'empty geojson-table should be geojson'); t.equal(emptyTable[0].data.rows.length, 0, 'should keep zero geojson-table features'); const emptyExcel = await processFileData({ content: {fileName: 'empty.xlsx', data: {shape: 'object-row-table', data: []}}, fileCache: [] }); t.equal(emptyExcel[0].info.format, 'row', 'empty spreadsheet tables should stay rows'); t.end(); }); test('#file-handler -> readBatch does not wrap JSON streaming placeholders', async t => { async function* geojsonStream() { yield {batchType: 'metadata', shape: 'metadata', data: []}; yield { shape: 'object-row-table', batchType: 'partial-result', container: {type: 'FeatureCollection', features: []}, data: [], jsonpath: '$.features' }; yield { data: [ { type: 'Feature', properties: {name: 'alpha'}, geometry: {type: 'Point', coordinates: [0, 1]} } ], jsonpath: '$.features', length: 1 }; } const generator = readBatch(geojsonStream(), 'places.geojson'); await generator.next(); const partial = await generator.next(); t.equal(partial.value.batchType, 'partial-result', 'should yield the JSON partial-result'); t.equal(partial.value.data.length, 0, 'partial-result should keep an empty array'); const dataBatch = await generator.next(); t.equal(dataBatch.value.data.length, 1, 'data batch should stay a feature array'); t.end(); }); test('#file-handler -> readBatch preserves empty loader identity', async t => { async function* emptyShapefileBatches() { yield {header: {length: 50}, data: []}; } const shapefileBatch = await readLastAggregatedBatch(emptyShapefileBatches(), 'empty.shp'); const emptyShapefile = await processFileData({content: shapefileBatch, fileCache: []}); t.equal(emptyShapefile[0].info.format, 'geojson', 'empty shapefile batches should stay geojson'); t.equal(emptyShapefile[0].data.rows.length, 0, 'should keep zero shapefile features'); async function* emptyExcelBatches() { yield {shape: 'object-row-table', data: []}; } const excelBatch = await readLastAggregatedBatch(emptyExcelBatches(), 'empty.xlsx'); const emptyExcel = await processFileData({content: excelBatch, fileCache: []}); t.equal(emptyExcel[0].info.format, 'row', 'empty spreadsheet batches should stay rows'); t.equal(emptyExcel[0].data.rows.length, 0, 'should keep zero spreadsheet rows'); async function* emptyFlatGeobufBatches() { yield {batchType: 'metadata'}; yield {shape: 'geojson-table', type: 'FeatureCollection', features: []}; } const fgbBatch = await readLastAggregatedBatch(emptyFlatGeobufBatches(), 'empty.fgb'); const emptyFgb = await processFileData({content: fgbBatch, fileCache: []}); t.equal(emptyFgb[0].info.format, 'geojson', 'empty FlatGeobuf batches should stay geojson'); t.equal(emptyFgb[0].data.rows.length, 0, 'should keep zero FlatGeobuf features'); t.end(); }); test('#file-handler -> object-row-table becomes a row dataset', async t => { const processed = await processFileData({ content: { fileName: 'table.xlsx', data: {shape: 'object-row-table', data: [{name: 'alpha', value: 1}]} }, fileCache: [] }); t.equal(processed[0].info.format, 'row', 'Excel-shaped tables should process as rows'); t.equal(processed[0].data.rows.length, 1, 'should keep the spreadsheet row'); t.end(); }); test('#file-handler -> rows with type Feature but no geometry stay tables', async t => { const rows = [ {type: 'Feature', name: 'alpha', lat: 37.8, lng: -122.4}, {type: 'Feature', name: 'beta', lat: 37.9, lng: -122.5} ]; t.equal( getGeoJsonFromLoaderResult(rows), null, 'should not wrap tabular rows as a FeatureCollection' ); const processed = await processFileData({ content: {fileName: 'places.csv', data: rows}, fileCache: [] }); t.equal( processed[0].info.format, 'row', 'a type column equal to Feature should not force geojson' ); t.equal(processed[0].data.rows.length, 2, 'should keep both rows'); t.end(); }); test('#file-handler -> processFileData skipArrowCompact', async t => { const batchA = arrow.tableFromJSON([{lng: -122.4, lat: 37.8}]); const batchB = arrow.tableFromJSON([{lng: -122.5, lat: 37.9}]); const batchC = arrow.tableFromJSON([{lng: -122.6, lat: 38.0}]); const combined = batchA.concat(batchB).concat(batchC); const incremental = await processFileData({ content: { fileName: 'points.arrow', data: combined.batches, skipArrowCompact: true }, fileCache: [] }); t.equal( incremental[0].data.cols[0].data.length, combined.batches.length, 'progressive processFileData should not compact Arrow batches' ); const finished = await processFileData({ content: { fileName: 'points.arrow', data: combined.batches }, fileCache: [] }); t.equal( finished[0].data.cols[0].data.length, 1, 'final processFileData should compact Arrow batches' ); const staged = await processFileData({ content: { fileName: 'points.arrow', data: combined.batches, prepareProgressiveReplay: true }, fileCache: [] }); t.equal(staged[0].progressiveData.length, 3, 'should keep one replay snapshot per batch'); t.deepEqual( staged[0].progressiveData.map(data => data.cols[0].length), [1, 2, 3], 'replay snapshots should grow by one batch at a time' ); t.deepEqual( staged[0].progressiveData.map(data => data.cols[0].data.length), [1, 2, 1], 'only the completed replay snapshot should be compacted' ); const csv = { data: {fields: [], rows: [[1]]}, info: {id: 'csv', label: 'points.csv', format: 'csv'} }; const steps = fileCacheToProgressiveSteps([staged[0], csv]); t.equal(steps.length, 3, 'should create one confirmed Add Data update per Arrow batch'); t.equal(steps[0].length, 2, 'non-progressive files should be included in the first update'); t.equal(steps[1].length, 1, 'later updates should contain only the Arrow dataset'); t.notOk(steps[0][0].progressiveData, 'internal replay data should not enter map state'); t.end(); });