import { pickDefaultPostStatus } from "@feeblo/domain-contracts/post-status-default"; import { DataImportJobId, type LegidError } from "@feeblo/id"; import * as Context from "effect/Context"; import { Digest } from "effect/crypto"; import * as Effect from "effect/Effect"; import * as Filter from "effect/Filter"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import { BoardRepository } from "../board/repository"; import * as Policy from "../policy"; import { PostStatusRepository } from "../post-status/repository"; import { InternalServerError, withRemapDbErrors } from "../rpc-errors"; import { CurrentSession } from "../session-middleware"; import { decodeBoardPostCsv, parseBoardPostCsv } from "./csv"; import type { ParsedBoardPostRow } from "./csv"; import { ACTIVE_IMPORT_MESSAGE, DataImportAlreadyActiveError, type DataImportProviderError, DataImportFileTooLargeError, DataImportNotConfirmableError, DataImportNotFoundError, DataImportRowLimitError, type DataTransferRepositoryError, InvalidBoardPostCsvError, UnknownDataImportProviderError, } from "./errors"; import { DATA_IMPORT_MAX_BYTES, DATA_IMPORT_MAX_ROWS, DATA_IMPORT_WATCH_MS, } from "./limits"; import { canConfirmDataImportStatus, isActiveDataImportStatus, } from "./machine"; import { planImportRows } from "./plan"; import { canImportPosts } from "./policies"; import { type ImportProviderCredentials, ImportProviderRegistry, } from "./provider"; import type { TDataImportProviderDescriptor, TDataImportProviderId, } from "./provider-schema"; import { type DataImportJobRecord, type DataImportReportSummary, type DataImportRowRecord, DataTransferRepository, type NewStagedImportRow, } from "./repository"; import { BOARD_IMPORT_DEFAULT_REPORT_LIMIT, BOARD_IMPORT_MAX_REPORT_LIMIT, type TDataImportRowFilter, type TDataImportRowSort, } from "./schema"; export type StageBoardImportInput = { readonly bytes: Uint8Array; readonly fileName: string; readonly organizationId: string; readonly userId: string; readonly memberId: string | null; }; export type DataImportJobDetail = { readonly job: DataImportJobRecord; readonly report: DataImportReportSummary & { readonly rows: readonly DataImportRowRecord[]; readonly total: number; }; /** The boards a staged file will create; empty once the job has left staging. */ readonly newBoards: readonly string[]; }; export type StageProviderImportInput = { readonly organizationId: string; readonly userId: string; readonly memberId: string | null; readonly provider: TDataImportProviderId; readonly credentials: ImportProviderCredentials; }; export type DataImportProviderErrors = | UnknownDataImportProviderError | DataImportProviderError; export type DescribeImportInput = { readonly organizationId: string; readonly id: string; readonly limit?: number | undefined; readonly offset?: number | undefined; readonly filter?: TDataImportRowFilter | undefined; readonly board?: string | undefined; readonly message?: string | undefined; readonly sort?: TDataImportRowSort | undefined; }; export type DataImportServiceContract = { readonly stageBoardImport: ( input: StageBoardImportInput ) => Effect.Effect< DataImportJobRecord, | InvalidBoardPostCsvError | DataImportRowLimitError | DataImportFileTooLargeError | DataImportAlreadyActiveError | Policy.PolicyDeniedError | InternalServerError | LegidError, CurrentSession >; /** * Pulls every post a provider credential can see and stages it through the * same planner as a CSV upload. Network work happens before the transaction, * so a slow provider never holds a database connection. Only an RPC exposes * this: provider credentials never travel over the dashboard's file routes. */ readonly stageProviderImport: ( input: StageProviderImportInput ) => Effect.Effect< DataImportJobRecord, | DataImportProviderErrors | DataImportRowLimitError | DataImportAlreadyActiveError | Policy.PolicyDeniedError | InternalServerError | LegidError, CurrentSession >; /** * The providers this deployment offers. The dashboard renders this list, so * adding a fourth provider is a server-side adapter rather than a release * of the web app. */ readonly listProviders: (input: { readonly organizationId: string; }) => Effect.Effect< readonly TDataImportProviderDescriptor[], Policy.PolicyDeniedError, CurrentSession >; readonly confirmImport: (input: { readonly organizationId: string; readonly id: string; }) => Effect.Effect< void, | DataImportNotFoundError | DataImportNotConfirmableError | Policy.PolicyDeniedError | InternalServerError, CurrentSession >; readonly cancelImport: (input: { readonly organizationId: string; readonly id: string; }) => Effect.Effect< void, DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError, CurrentSession >; readonly describeImport: ( input: DescribeImportInput ) => Effect.Effect< DataImportJobDetail, DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError, CurrentSession >; readonly listImports: (input: { readonly organizationId: string; }) => Effect.Effect< readonly DataImportJobRecord[], Policy.PolicyDeniedError | InternalServerError, CurrentSession >; /** * The job's current snapshot, re-read on an interval and re-emitted only * when something a viewer sees changed. Ends after the first terminal * snapshot: a finished job has nothing left to report. */ readonly watchImport: ( input: DescribeImportInput ) => Stream.Stream< DataImportJobDetail, DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError, CurrentSession >; /** * The workspace's job list, re-read on an interval and re-emitted only when * a job row changed. The stream stays open: another upload can arrive at * any time. */ readonly watchImportList: (input: { readonly organizationId: string; }) => Stream.Stream< readonly DataImportJobRecord[], Policy.PolicyDeniedError | InternalServerError, CurrentSession >; }; /** The most recent jobs a workspace sees, newest first. */ const IMPORT_JOB_LIST_LIMIT = 20; const textEncoder = new TextEncoder(); /** Normalizes a store failure to one stable internal failure. */ const fromStore = ( operation: string, effect: Effect.Effect ): Effect.Effect => effect.pipe( Effect.catchTag("DataTransferRepositoryError", () => Effect.fail( new InternalServerError({ message: `The import store failed during ${operation}.`, }) ) ) ); const clampReportWindow = ( limit: number | undefined, offset: number | undefined ) => ({ limit: Math.min( Math.max(limit ?? BOARD_IMPORT_DEFAULT_REPORT_LIMIT, 1), BOARD_IMPORT_MAX_REPORT_LIMIT ), offset: Math.max(offset ?? 0, 0), }); /** * A cheap key over the fields of a job a viewer sees change while it runs. * Two snapshots with the same key are the same picture, so a watch tick that * finds nothing new stays silent instead of waking every subscriber. */ const jobChangeKey = (job: DataImportJobRecord): string => [ job.id, job.status, job.createdCount, job.warningCount, job.errorCount, job.finishedAt?.getTime() ?? 0, job.failureMessage ?? "", ].join("\u0000"); /** The job key plus the visible shape of the one report page being watched. */ const reportChangeKey = (detail: DataImportJobDetail): string => [ jobChangeKey(detail.job), detail.report.total, detail.report.counts.total, detail.report.counts.pending, detail.report.counts.created, detail.report.counts.failed, detail.report.counts.warned, detail.newBoards.length, ...detail.report.rows.map( (row) => `${row.id}\u0000${row.outcome}\u0000${row.postId ?? ""}\u0000${row.warnings.length}` ), ].join("\u0001"); /** Drops stream values whose change key matches the previous emission. */ const excludeUnchanged = (keyOf: (value: A) => string) => (self: Stream.Stream): Stream.Stream => self.pipe( Stream.mapAccum( () => "", (previous: string, value: A): readonly [string, ReadonlyArray] => { const key = keyOf(value); return key === previous ? [previous, []] : [key, [value]]; } ) ); /** * One watch tick. An infrastructure failure is not a reason to end a watch * that the next tick may recover from — the same stance the worker's pass * loop takes — so it suppresses that tick and lets the schedule retry. The * typed domain failures (denied, not found) stay fatal: retrying those would * spin forever without a way out. */ const suppressInternalFailures = ( effect: Effect.Effect ): Effect.Effect, E, R> => effect.pipe( Effect.asSome, Effect.catchTag("InternalServerError", () => Effect.succeed(Option.none()) ) ); const makeDataImportService = Effect.gen(function* () { const repository = yield* DataTransferRepository; const boards = yield* BoardRepository; const statuses = yield* PostStatusRepository; const digest = yield* Digest.Digest; const importProviders = yield* ImportProviderRegistry; /** SHA-256 of the uploaded file, hex encoded; the import's idempotency key. */ const hashBytes = (bytes: Uint8Array) => digest.digestHex(Digest.sha256, bytes); /** * The fingerprint of a provider pull is text, not a file. Hashing it gives a * provider import the same duplicate-upload warning a CSV gets without a * second identity column. */ const hashText = (value: string) => digest.digestHex(Digest.sha256, textEncoder.encode(value)); /** * The workspace's one active-import slot. The unique index is the authority; * this is the fast path, shared by both staging entry points so a provider * pull and a file upload cannot disagree about when a workspace is busy. */ const requireNoActiveImport = (organizationId: string) => Effect.gen(function* () { const hasActive = yield* fromStore( "hasActiveJob", repository.hasActiveJob({ organizationId }) ); if (hasActive) { return yield* new DataImportAlreadyActiveError({ message: ACTIVE_IMPORT_MESSAGE, }); } }); type StageParsedRowsInput = { readonly organizationId: string; readonly userId: string; readonly memberId: string | null; readonly fileName: string; readonly fileHash: string; readonly parsed: { readonly rows: readonly ParsedBoardPostRow[]; readonly notices: readonly string[]; }; readonly duplicateNotice: (importedAt: Date) => string; }; /** * The shared plan-and-stage step. A CSV upload and a provider pull differ in * how they produce rows and a fingerprint; from there the board/status * resolution, the row report and the confirmation gate are one pipeline, so * a provider can never invent a second interpretation of a staged row. */ const stageParsedRows = (input: StageParsedRowsInput) => Effect.gen(function* () { if (input.parsed.rows.length > DATA_IMPORT_MAX_ROWS) { return yield* new DataImportRowLimitError({ maxRows: DATA_IMPORT_MAX_ROWS, message: `The import has ${input.parsed.rows.length} rows; the limit is ${DATA_IMPORT_MAX_ROWS}.`, }); } // Imports are not scoped to a board: the `board` column routes every // row. The oldest board is the fallback for a row with an empty cell, // and is null only when the workspace has no boards at all. The read // and the sort happen here so the plan stays a pure function of them. const [existingBoards, statusRows] = yield* Effect.all( [ boards .findMany({ organizationId: input.organizationId }) .pipe(withRemapDbErrors("Board", "select")), statuses .findMany({ organizationId: input.organizationId }) .pipe(withRemapDbErrors("PostStatus", "select")), ], { concurrency: "unbounded" } ); const sortedBoards = [...existingBoards].sort( (left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id) ); const [defaultBoard = null] = sortedBoards; // The workspace's stored default, not the `PENDING` type: a workspace may // repoint which status is the default, and several statuses may share a // type (docs/adr/0013-a-status-default-is-stored-and-a-type-may-hold-several). // The name the row report shows for a substituted status must be the one // the worker's rows actually land in. const defaultStatus = pickDefaultPostStatus(statusRows); if (defaultStatus === undefined) { return yield* new InternalServerError({ message: "This workspace has no post statuses. Add one before importing posts.", }); } const prior = yield* fromStore( "findPriorJobByHash", repository.findPriorJobByHash({ fileHash: input.fileHash, organizationId: input.organizationId, }) ); const plan = planImportRows({ boards: sortedBoards, defaultBoard, defaultStatus, rows: input.parsed.rows, statuses: statusRows, }); const priorNotices = Option.match(prior, { onNone: (): readonly string[] => [], onSome: (job) => [input.duplicateNotice(job.createdAt)], }); const rows: readonly NewStagedImportRow[] = plan.rows.map((row) => row.kind === "pending" ? { message: null, outcome: "pending", payload: row.payload, rowNumber: row.rowNumber, } : { message: row.message, outcome: "failed", payload: null, rowNumber: row.rowNumber, } ); const jobId = yield* DataImportJobId.generate; return yield* repository .insertStagedJob({ createdByMemberId: input.memberId, createdByUserId: input.userId, fileName: input.fileName, fileHash: input.fileHash, id: jobId, notices: [...input.parsed.notices, ...plan.notices, ...priorNotices], organizationId: input.organizationId, rows, }) .pipe( Effect.catchTag("DataTransferRepositoryError", () => Effect.fail( new InternalServerError({ message: "The import store failed during insertStagedJob.", }) ) ) ); }); const stageBoardImport: DataImportServiceContract["stageBoardImport"] = ( input ) => Effect.gen(function* () { if (input.bytes.byteLength > DATA_IMPORT_MAX_BYTES) { return yield* new DataImportFileTooLargeError({ maxBytes: DATA_IMPORT_MAX_BYTES, message: `The file is larger than ${Math.round( DATA_IMPORT_MAX_BYTES / (1024 * 1024) )} MB.`, }); } yield* requireNoActiveImport(input.organizationId); const [text, fileHash] = yield* Effect.all( [decodeBoardPostCsv(input.bytes), hashBytes(input.bytes)], { concurrency: "unbounded" } ); const parsed = yield* parseBoardPostCsv(text); return yield* stageParsedRows({ duplicateNotice: (importedAt) => `This file was already imported on ${importedAt .toISOString() .slice(0, 10)}.`, fileName: input.fileName, fileHash, memberId: input.memberId, organizationId: input.organizationId, parsed, userId: input.userId, }); }).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const stageProviderImport: DataImportServiceContract["stageProviderImport"] = (input) => Effect.gen(function* () { const adapter = importProviders.find(input.provider); if (Option.isNone(adapter)) { return yield* new UnknownDataImportProviderError({ message: `The "${input.provider}" import source is not available in this deployment.`, provider: input.provider, }); } // Check the slot before the pull, not after: a crawler that takes // minutes should not run for a workspace that cannot stage its result. yield* requireNoActiveImport(input.organizationId); const result = yield* adapter.value.fetchPosts({ credentials: input.credentials, }); const fileHash = yield* hashText(result.fingerprint); return yield* stageParsedRows({ duplicateNotice: (importedAt) => `These posts were already imported on ${importedAt .toISOString() .slice(0, 10)}.`, fileName: `${adapter.value.descriptor.name} import`, fileHash, memberId: input.memberId, organizationId: input.organizationId, parsed: { notices: result.notices, rows: result.rows }, userId: input.userId, }); }).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const listProviders: DataImportServiceContract["listProviders"] = (input) => Effect.sync(() => importProviders.list()).pipe( Policy.withPolicy(canImportPosts(input.organizationId)) ); const confirmImport: DataImportServiceContract["confirmImport"] = (input) => Effect.gen(function* () { const queued = yield* fromStore( "markQueued", repository.markQueued({ id: input.id, organizationId: input.organizationId, }) ); if (Option.isSome(queued)) { return; } const job = yield* fromStore( "findJob", repository.findJob({ id: input.id, organizationId: input.organizationId, }) ); if (Option.isNone(job)) { return yield* new DataImportNotFoundError({ message: "No import with this id exists in this workspace.", }); } return yield* new DataImportNotConfirmableError({ status: job.value.status, message: `This import is ${job.value.status.replaceAll("_", " ")} and cannot be confirmed.`, }); }).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const cancelImport: DataImportServiceContract["cancelImport"] = (input) => Effect.gen(function* () { const canceled = yield* fromStore( "cancelJob", repository.cancelJob({ id: input.id, organizationId: input.organizationId, }) ); if (canceled) { return; } const job = yield* fromStore( "findJob", repository.findJob({ id: input.id, organizationId: input.organizationId, }) ); if (Option.isNone(job)) { return yield* new DataImportNotFoundError({ message: "No import with this id exists in this workspace.", }); } // A finished or already-canceled job is not an error: cancel is // idempotent, and the caller's intent is satisfied either way. }).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const describeImport: DataImportServiceContract["describeImport"] = (input) => Effect.gen(function* () { const job = yield* fromStore( "findJob", repository.findJob({ id: input.id, organizationId: input.organizationId, }) ); if (Option.isNone(job)) { return yield* new DataImportNotFoundError({ message: "No import with this id exists in this workspace.", }); } const window = clampReportWindow(input.limit, input.offset); // The row window, its summary, and the pending board names are three // reads of the same staged job and none needs another's answer, so they // resolve together; the report is polled while an import runs, which is // exactly when it should not cost three round trips. const [report, summary, newBoards] = yield* Effect.all( [ fromStore( "listRows", repository.listRows({ board: input.board, filter: input.filter, jobId: input.id, limit: window.limit, message: input.message, offset: window.offset, sort: input.sort, }) ), fromStore( "summarizeReport", repository.summarizeReport({ jobId: input.id }) ), // A staged file may name boards that do not exist yet, and the // confirmation names them. Once the job runs, the worker creates // them, so an unchanged list would only be stale. canConfirmDataImportStatus(job.value.status) ? fromStore( "listNewBoardNames", repository.listNewBoardNames({ jobId: input.id }) ) : Effect.succeed([]), ], { concurrency: "unbounded" } ); return { job: job.value, newBoards: [...newBoards], report: { boardGroups: summary.boardGroups, counts: summary.counts, failureGroups: summary.failureGroups, rows: report.rows, total: report.total, }, }; }).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const listImports: DataImportServiceContract["listImports"] = (input) => fromStore( "listJobs", repository.listJobs({ limit: IMPORT_JOB_LIST_LIMIT, organizationId: input.organizationId, }) ).pipe(Policy.withPolicy(canImportPosts(input.organizationId))); const watchImport: DataImportServiceContract["watchImport"] = (input) => Stream.fromEffectSchedule( suppressInternalFailures(describeImport(input)), Schedule.spaced(DATA_IMPORT_WATCH_MS) ).pipe( Stream.filterMap( Filter.fromPredicateOption( (option: Option.Option) => option ) ), // A job still moves through the active statuses on its own; everything // else is final. This watch writes its last snapshot and ends on the // transition out — the predicate is the machine's, the same graph every // guarded `UPDATE` in the repository asks. Stream.takeUntil( (detail) => !isActiveDataImportStatus(detail.job.status) ), excludeUnchanged(reportChangeKey) ); const watchImportList: DataImportServiceContract["watchImportList"] = ( input ) => Stream.fromEffectSchedule( suppressInternalFailures( listImports({ organizationId: input.organizationId }) ), Schedule.spaced(DATA_IMPORT_WATCH_MS) ).pipe( Stream.filterMap( Filter.fromPredicateOption( (option: Option.Option) => option ) ), excludeUnchanged((jobs) => jobs.map(jobChangeKey).join("\u0001")) ); return { cancelImport, confirmImport, describeImport, listImports, listProviders, stageBoardImport, stageProviderImport, watchImport, watchImportList, } satisfies DataImportServiceContract; }); export class DataImportService extends Context.Service< DataImportService, DataImportServiceContract >()("DataImportService", { make: makeDataImportService, }) { static readonly layer = Layer.effect(this, this.make).pipe( Layer.provide(DataTransferRepository.layer), Layer.provide(BoardRepository.layer), Layer.provide(PostStatusRepository.layer) ); }