import { deserialize, registerSerializableClass, serialize, } from '@atlaspack/build-cache'; import {Lmdb} from '@atlaspack/rust'; import type {FilePath} from '@atlaspack/types'; import type {Cache} from './types'; import type {Readable, Writable} from 'stream'; // @ts-expect-error TS7016 import ncp from 'ncp'; import {promisify} from 'util'; import stream from 'stream'; import path from 'path'; import {NodeFS} from '@atlaspack/fs'; import packageJson from '../package.json'; import {FSCache} from './FSCache'; import {instrumentAsync} from '@atlaspack/logger'; const ncpAsync = promisify(ncp); interface DBOpenOptions { name: string; // unused encoding: string; // unused compression: boolean; } export class LmdbWrapper { lmdb: Lmdb; constructor(lmdb: Lmdb) { this.lmdb = lmdb; } has(key: string): boolean { return this.lmdb.hasSync(key); } async delete(key: string): Promise { await this.lmdb.delete(key); } get(key: string): Buffer | null { return this.lmdb.getSync(key); } async put(key: string, value: Buffer | string): Promise { const buffer: Buffer = typeof value === 'string' ? Buffer.from(value) : value; await this.lmdb.put(key, buffer); } *keys(): Iterable { const PAGE_SIZE = 10000000; let currentKeys = this.lmdb.keysSync(0, PAGE_SIZE); while (currentKeys.length > 0) { for (const key of currentKeys) { yield key; } currentKeys = this.lmdb.keysSync(currentKeys.length, PAGE_SIZE); } } compact(targetPath: string) { this.lmdb.compact(targetPath); } } export function open( directory: string, // eslint-disable-next-line no-unused-vars openOptions: DBOpenOptions, ): LmdbWrapper { return new LmdbWrapper( new Lmdb({ path: directory, asyncWrites: true, mapSize: process.env.ATLASPACK_BUILD_ENV === 'test' ? 256 * 1024 * 1024 : 1024 * 1024 * 1024 * 15, }), ); } const pipeline: (arg1: Readable, arg2: Writable) => Promise = promisify( stream.pipeline, ); export type SerLMDBLiteCache = { dir: FilePath; }; export class LMDBLiteCache implements Cache { fs: NodeFS; dir: FilePath; store: LmdbWrapper; fsCache: FSCache; /** * Directory where we store raw files. */ cacheFilesDirectory: FilePath; constructor(cacheDir: FilePath) { this.fs = new NodeFS(); this.dir = cacheDir; this.cacheFilesDirectory = path.join(cacheDir, 'files'); this.fsCache = new FSCache(this.fs, cacheDir); this.store = open(cacheDir, { name: 'parcel-cache', encoding: 'binary', compression: true, }); } /** * Use this to pass the native LMDB instance back to Rust. */ getNativeRef(): Lmdb { return this.store.lmdb; } async ensure(): Promise { await this.fsCache.ensure(); await this.fs.mkdirp(this.cacheFilesDirectory); return Promise.resolve(); } serialize(): SerLMDBLiteCache { return { dir: this.dir, }; } static deserialize(cache: SerLMDBLiteCache): LMDBLiteCache { return new LMDBLiteCache(cache.dir); } has(key: string): Promise { return Promise.resolve(this.store.has(key)); } get(key: string): Promise { let data = this.store.get(key); if (data == null) { return Promise.resolve(null); } return Promise.resolve(deserialize(data)); } async set(key: string, value: unknown): Promise { await this.setBlob(key, serialize(value)); } getStream(key: string): Readable { return this.fs.createReadStream(path.join(this.dir, key)); } setStream(key: string, stream: Readable): Promise { return pipeline( stream, this.fs.createWriteStream(path.join(this.dir, key)), ); } // eslint-disable-next-line require-await async getBlob(key: string): Promise { return this.getBlobSync(key); } getBlobSync(key: string): Buffer { const buffer = this.store.get(key); if (buffer == null) { throw new Error(`Key ${key} not found in cache`); } return buffer; } async setBlob(key: string, contents: Buffer | string): Promise { await this.store.put(key, contents); } getBuffer(key: string): Promise { return Promise.resolve(this.store.get(key)); } hasLargeBlob(key: string): Promise { return this.fsCache.hasLargeBlob(key); } getLargeBlob(key: string): Promise { return this.fsCache.getLargeBlob(key); } setLargeBlob( key: string, contents: Buffer | string, options?: { signal?: AbortSignal; }, ): Promise { return this.fsCache.setLargeBlob(key, contents, options); } /** * @deprecated Use store.delete instead. */ deleteLargeBlob(key: string): Promise { return this.fsCache.deleteLargeBlob(key); } keys(): Iterable { return this.store.keys(); } async compact(targetPath: string): Promise { await this.fs.mkdirp(targetPath); const files = await this.fs.readdir(this.dir); // copy all files except data.mdb and lock.mdb to the target path (recursive) for (const file of files) { const filePath = path.join(this.dir, file); if (file === 'data.mdb' || file === 'lock.mdb') { continue; } await ncpAsync(filePath, path.join(targetPath, file)); } this.store.compact(path.join(targetPath, 'data.mdb')); } refresh(): void {} /** * Streams, packages are stored in files instead of LMDB. * * On this case, if a cache key happens to have a parent traversal, ../.. * it is treated specially * * That is, something/../something and something are meant to be different * keys. * * Plus we do not want to store values outside of the cache directory. */ getFileKey(key: string): string { const cleanKey = key .split('/') .map((part) => { if (part === '..') { return '$$__parent_dir$$'; } return part; }) .join('/'); return path.join(this.cacheFilesDirectory, cleanKey); } async clear(): Promise { await instrumentAsync('LMDBLiteCache::clear', async () => { const keys = await this.keys(); for (const key of keys) { await this.store.delete(key); } await this.fs.rimraf(this.cacheFilesDirectory); await this.fs.mkdirp(this.cacheFilesDirectory); }); } } registerSerializableClass( `${packageJson.version}:LMDBLiteCache`, LMDBLiteCache, );