import { pickDefaultPostStatus } from "@feeblo/domain-contracts/post-status-default";
import { DataImportJobId, type LegidError } from "@feeblo/id";
import * as Context from "effect/Context";
import { Digest } from "effect/crypto";
import * as Effect from "effect/Effect";
import * as Filter from "effect/Filter";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { BoardRepository } from "../board/repository";
import * as Policy from "../policy";
import { PostStatusRepository } from "../post-status/repository";
import { InternalServerError, withRemapDbErrors } from "../rpc-errors";
import { CurrentSession } from "../session-middleware";
import { decodeBoardPostCsv, parseBoardPostCsv } from "./csv";
import type { ParsedBoardPostRow } from "./csv";
import {
ACTIVE_IMPORT_MESSAGE,
DataImportAlreadyActiveError,
type DataImportProviderError,
DataImportFileTooLargeError,
DataImportNotConfirmableError,
DataImportNotFoundError,
DataImportRowLimitError,
type DataTransferRepositoryError,
InvalidBoardPostCsvError,
UnknownDataImportProviderError,
} from "./errors";
import {
DATA_IMPORT_MAX_BYTES,
DATA_IMPORT_MAX_ROWS,
DATA_IMPORT_WATCH_MS,
} from "./limits";
import {
canConfirmDataImportStatus,
isActiveDataImportStatus,
} from "./machine";
import { planImportRows } from "./plan";
import { canImportPosts } from "./policies";
import {
type ImportProviderCredentials,
ImportProviderRegistry,
} from "./provider";
import type {
TDataImportProviderDescriptor,
TDataImportProviderId,
} from "./provider-schema";
import {
type DataImportJobRecord,
type DataImportReportSummary,
type DataImportRowRecord,
DataTransferRepository,
type NewStagedImportRow,
} from "./repository";
import {
BOARD_IMPORT_DEFAULT_REPORT_LIMIT,
BOARD_IMPORT_MAX_REPORT_LIMIT,
type TDataImportRowFilter,
type TDataImportRowSort,
} from "./schema";
export type StageBoardImportInput = {
readonly bytes: Uint8Array;
readonly fileName: string;
readonly organizationId: string;
readonly userId: string;
readonly memberId: string | null;
};
export type DataImportJobDetail = {
readonly job: DataImportJobRecord;
readonly report: DataImportReportSummary & {
readonly rows: readonly DataImportRowRecord[];
readonly total: number;
};
/** The boards a staged file will create; empty once the job has left staging. */
readonly newBoards: readonly string[];
};
export type StageProviderImportInput = {
readonly organizationId: string;
readonly userId: string;
readonly memberId: string | null;
readonly provider: TDataImportProviderId;
readonly credentials: ImportProviderCredentials;
};
export type DataImportProviderErrors =
| UnknownDataImportProviderError
| DataImportProviderError;
export type DescribeImportInput = {
readonly organizationId: string;
readonly id: string;
readonly limit?: number | undefined;
readonly offset?: number | undefined;
readonly filter?: TDataImportRowFilter | undefined;
readonly board?: string | undefined;
readonly message?: string | undefined;
readonly sort?: TDataImportRowSort | undefined;
};
export type DataImportServiceContract = {
readonly stageBoardImport: (
input: StageBoardImportInput
) => Effect.Effect<
DataImportJobRecord,
| InvalidBoardPostCsvError
| DataImportRowLimitError
| DataImportFileTooLargeError
| DataImportAlreadyActiveError
| Policy.PolicyDeniedError
| InternalServerError
| LegidError,
CurrentSession
>;
/**
* Pulls every post a provider credential can see and stages it through the
* same planner as a CSV upload. Network work happens before the transaction,
* so a slow provider never holds a database connection. Only an RPC exposes
* this: provider credentials never travel over the dashboard's file routes.
*/
readonly stageProviderImport: (
input: StageProviderImportInput
) => Effect.Effect<
DataImportJobRecord,
| DataImportProviderErrors
| DataImportRowLimitError
| DataImportAlreadyActiveError
| Policy.PolicyDeniedError
| InternalServerError
| LegidError,
CurrentSession
>;
/**
* The providers this deployment offers. The dashboard renders this list, so
* adding a fourth provider is a server-side adapter rather than a release
* of the web app.
*/
readonly listProviders: (input: {
readonly organizationId: string;
}) => Effect.Effect<
readonly TDataImportProviderDescriptor[],
Policy.PolicyDeniedError,
CurrentSession
>;
readonly confirmImport: (input: {
readonly organizationId: string;
readonly id: string;
}) => Effect.Effect<
void,
| DataImportNotFoundError
| DataImportNotConfirmableError
| Policy.PolicyDeniedError
| InternalServerError,
CurrentSession
>;
readonly cancelImport: (input: {
readonly organizationId: string;
readonly id: string;
}) => Effect.Effect<
void,
DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError,
CurrentSession
>;
readonly describeImport: (
input: DescribeImportInput
) => Effect.Effect<
DataImportJobDetail,
DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError,
CurrentSession
>;
readonly listImports: (input: {
readonly organizationId: string;
}) => Effect.Effect<
readonly DataImportJobRecord[],
Policy.PolicyDeniedError | InternalServerError,
CurrentSession
>;
/**
* The job's current snapshot, re-read on an interval and re-emitted only
* when something a viewer sees changed. Ends after the first terminal
* snapshot: a finished job has nothing left to report.
*/
readonly watchImport: (
input: DescribeImportInput
) => Stream.Stream<
DataImportJobDetail,
DataImportNotFoundError | Policy.PolicyDeniedError | InternalServerError,
CurrentSession
>;
/**
* The workspace's job list, re-read on an interval and re-emitted only when
* a job row changed. The stream stays open: another upload can arrive at
* any time.
*/
readonly watchImportList: (input: {
readonly organizationId: string;
}) => Stream.Stream<
readonly DataImportJobRecord[],
Policy.PolicyDeniedError | InternalServerError,
CurrentSession
>;
};
/** The most recent jobs a workspace sees, newest first. */
const IMPORT_JOB_LIST_LIMIT = 20;
const textEncoder = new TextEncoder();
/** Normalizes a store failure to one stable internal failure. */
const fromStore = (
operation: string,
effect: Effect.Effect
): Effect.Effect =>
effect.pipe(
Effect.catchTag("DataTransferRepositoryError", () =>
Effect.fail(
new InternalServerError({
message: `The import store failed during ${operation}.`,
})
)
)
);
const clampReportWindow = (
limit: number | undefined,
offset: number | undefined
) => ({
limit: Math.min(
Math.max(limit ?? BOARD_IMPORT_DEFAULT_REPORT_LIMIT, 1),
BOARD_IMPORT_MAX_REPORT_LIMIT
),
offset: Math.max(offset ?? 0, 0),
});
/**
* A cheap key over the fields of a job a viewer sees change while it runs.
* Two snapshots with the same key are the same picture, so a watch tick that
* finds nothing new stays silent instead of waking every subscriber.
*/
const jobChangeKey = (job: DataImportJobRecord): string =>
[
job.id,
job.status,
job.createdCount,
job.warningCount,
job.errorCount,
job.finishedAt?.getTime() ?? 0,
job.failureMessage ?? "",
].join("\u0000");
/** The job key plus the visible shape of the one report page being watched. */
const reportChangeKey = (detail: DataImportJobDetail): string =>
[
jobChangeKey(detail.job),
detail.report.total,
detail.report.counts.total,
detail.report.counts.pending,
detail.report.counts.created,
detail.report.counts.failed,
detail.report.counts.warned,
detail.newBoards.length,
...detail.report.rows.map(
(row) =>
`${row.id}\u0000${row.outcome}\u0000${row.postId ?? ""}\u0000${row.warnings.length}`
),
].join("\u0001");
/** Drops stream values whose change key matches the previous emission. */
const excludeUnchanged =
(keyOf: (value: A) => string) =>
(self: Stream.Stream): Stream.Stream =>
self.pipe(
Stream.mapAccum(
() => "",
(previous: string, value: A): readonly [string, ReadonlyArray] => {
const key = keyOf(value);
return key === previous ? [previous, []] : [key, [value]];
}
)
);
/**
* One watch tick. An infrastructure failure is not a reason to end a watch
* that the next tick may recover from — the same stance the worker's pass
* loop takes — so it suppresses that tick and lets the schedule retry. The
* typed domain failures (denied, not found) stay fatal: retrying those would
* spin forever without a way out.
*/
const suppressInternalFailures = (
effect: Effect.Effect
): Effect.Effect, E, R> =>
effect.pipe(
Effect.asSome,
Effect.catchTag("InternalServerError", () =>
Effect.succeed(Option.none())
)
);
const makeDataImportService = Effect.gen(function* () {
const repository = yield* DataTransferRepository;
const boards = yield* BoardRepository;
const statuses = yield* PostStatusRepository;
const digest = yield* Digest.Digest;
const importProviders = yield* ImportProviderRegistry;
/** SHA-256 of the uploaded file, hex encoded; the import's idempotency key. */
const hashBytes = (bytes: Uint8Array) =>
digest.digestHex(Digest.sha256, bytes);
/**
* The fingerprint of a provider pull is text, not a file. Hashing it gives a
* provider import the same duplicate-upload warning a CSV gets without a
* second identity column.
*/
const hashText = (value: string) =>
digest.digestHex(Digest.sha256, textEncoder.encode(value));
/**
* The workspace's one active-import slot. The unique index is the authority;
* this is the fast path, shared by both staging entry points so a provider
* pull and a file upload cannot disagree about when a workspace is busy.
*/
const requireNoActiveImport = (organizationId: string) =>
Effect.gen(function* () {
const hasActive = yield* fromStore(
"hasActiveJob",
repository.hasActiveJob({ organizationId })
);
if (hasActive) {
return yield* new DataImportAlreadyActiveError({
message: ACTIVE_IMPORT_MESSAGE,
});
}
});
type StageParsedRowsInput = {
readonly organizationId: string;
readonly userId: string;
readonly memberId: string | null;
readonly fileName: string;
readonly fileHash: string;
readonly parsed: {
readonly rows: readonly ParsedBoardPostRow[];
readonly notices: readonly string[];
};
readonly duplicateNotice: (importedAt: Date) => string;
};
/**
* The shared plan-and-stage step. A CSV upload and a provider pull differ in
* how they produce rows and a fingerprint; from there the board/status
* resolution, the row report and the confirmation gate are one pipeline, so
* a provider can never invent a second interpretation of a staged row.
*/
const stageParsedRows = (input: StageParsedRowsInput) =>
Effect.gen(function* () {
if (input.parsed.rows.length > DATA_IMPORT_MAX_ROWS) {
return yield* new DataImportRowLimitError({
maxRows: DATA_IMPORT_MAX_ROWS,
message: `The import has ${input.parsed.rows.length} rows; the limit is ${DATA_IMPORT_MAX_ROWS}.`,
});
}
// Imports are not scoped to a board: the `board` column routes every
// row. The oldest board is the fallback for a row with an empty cell,
// and is null only when the workspace has no boards at all. The read
// and the sort happen here so the plan stays a pure function of them.
const [existingBoards, statusRows] = yield* Effect.all(
[
boards
.findMany({ organizationId: input.organizationId })
.pipe(withRemapDbErrors("Board", "select")),
statuses
.findMany({ organizationId: input.organizationId })
.pipe(withRemapDbErrors("PostStatus", "select")),
],
{ concurrency: "unbounded" }
);
const sortedBoards = [...existingBoards].sort(
(left, right) =>
left.createdAt.getTime() - right.createdAt.getTime() ||
left.id.localeCompare(right.id)
);
const [defaultBoard = null] = sortedBoards;
// The workspace's stored default, not the `PENDING` type: a workspace may
// repoint which status is the default, and several statuses may share a
// type (docs/adr/0013-a-status-default-is-stored-and-a-type-may-hold-several).
// The name the row report shows for a substituted status must be the one
// the worker's rows actually land in.
const defaultStatus = pickDefaultPostStatus(statusRows);
if (defaultStatus === undefined) {
return yield* new InternalServerError({
message:
"This workspace has no post statuses. Add one before importing posts.",
});
}
const prior = yield* fromStore(
"findPriorJobByHash",
repository.findPriorJobByHash({
fileHash: input.fileHash,
organizationId: input.organizationId,
})
);
const plan = planImportRows({
boards: sortedBoards,
defaultBoard,
defaultStatus,
rows: input.parsed.rows,
statuses: statusRows,
});
const priorNotices = Option.match(prior, {
onNone: (): readonly string[] => [],
onSome: (job) => [input.duplicateNotice(job.createdAt)],
});
const rows: readonly NewStagedImportRow[] = plan.rows.map((row) =>
row.kind === "pending"
? {
message: null,
outcome: "pending",
payload: row.payload,
rowNumber: row.rowNumber,
}
: {
message: row.message,
outcome: "failed",
payload: null,
rowNumber: row.rowNumber,
}
);
const jobId = yield* DataImportJobId.generate;
return yield* repository
.insertStagedJob({
createdByMemberId: input.memberId,
createdByUserId: input.userId,
fileName: input.fileName,
fileHash: input.fileHash,
id: jobId,
notices: [...input.parsed.notices, ...plan.notices, ...priorNotices],
organizationId: input.organizationId,
rows,
})
.pipe(
Effect.catchTag("DataTransferRepositoryError", () =>
Effect.fail(
new InternalServerError({
message: "The import store failed during insertStagedJob.",
})
)
)
);
});
const stageBoardImport: DataImportServiceContract["stageBoardImport"] = (
input
) =>
Effect.gen(function* () {
if (input.bytes.byteLength > DATA_IMPORT_MAX_BYTES) {
return yield* new DataImportFileTooLargeError({
maxBytes: DATA_IMPORT_MAX_BYTES,
message: `The file is larger than ${Math.round(
DATA_IMPORT_MAX_BYTES / (1024 * 1024)
)} MB.`,
});
}
yield* requireNoActiveImport(input.organizationId);
const [text, fileHash] = yield* Effect.all(
[decodeBoardPostCsv(input.bytes), hashBytes(input.bytes)],
{ concurrency: "unbounded" }
);
const parsed = yield* parseBoardPostCsv(text);
return yield* stageParsedRows({
duplicateNotice: (importedAt) =>
`This file was already imported on ${importedAt
.toISOString()
.slice(0, 10)}.`,
fileName: input.fileName,
fileHash,
memberId: input.memberId,
organizationId: input.organizationId,
parsed,
userId: input.userId,
});
}).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const stageProviderImport: DataImportServiceContract["stageProviderImport"] =
(input) =>
Effect.gen(function* () {
const adapter = importProviders.find(input.provider);
if (Option.isNone(adapter)) {
return yield* new UnknownDataImportProviderError({
message: `The "${input.provider}" import source is not available in this deployment.`,
provider: input.provider,
});
}
// Check the slot before the pull, not after: a crawler that takes
// minutes should not run for a workspace that cannot stage its result.
yield* requireNoActiveImport(input.organizationId);
const result = yield* adapter.value.fetchPosts({
credentials: input.credentials,
});
const fileHash = yield* hashText(result.fingerprint);
return yield* stageParsedRows({
duplicateNotice: (importedAt) =>
`These posts were already imported on ${importedAt
.toISOString()
.slice(0, 10)}.`,
fileName: `${adapter.value.descriptor.name} import`,
fileHash,
memberId: input.memberId,
organizationId: input.organizationId,
parsed: { notices: result.notices, rows: result.rows },
userId: input.userId,
});
}).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const listProviders: DataImportServiceContract["listProviders"] = (input) =>
Effect.sync(() => importProviders.list()).pipe(
Policy.withPolicy(canImportPosts(input.organizationId))
);
const confirmImport: DataImportServiceContract["confirmImport"] = (input) =>
Effect.gen(function* () {
const queued = yield* fromStore(
"markQueued",
repository.markQueued({
id: input.id,
organizationId: input.organizationId,
})
);
if (Option.isSome(queued)) {
return;
}
const job = yield* fromStore(
"findJob",
repository.findJob({
id: input.id,
organizationId: input.organizationId,
})
);
if (Option.isNone(job)) {
return yield* new DataImportNotFoundError({
message: "No import with this id exists in this workspace.",
});
}
return yield* new DataImportNotConfirmableError({
status: job.value.status,
message: `This import is ${job.value.status.replaceAll("_", " ")} and cannot be confirmed.`,
});
}).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const cancelImport: DataImportServiceContract["cancelImport"] = (input) =>
Effect.gen(function* () {
const canceled = yield* fromStore(
"cancelJob",
repository.cancelJob({
id: input.id,
organizationId: input.organizationId,
})
);
if (canceled) {
return;
}
const job = yield* fromStore(
"findJob",
repository.findJob({
id: input.id,
organizationId: input.organizationId,
})
);
if (Option.isNone(job)) {
return yield* new DataImportNotFoundError({
message: "No import with this id exists in this workspace.",
});
}
// A finished or already-canceled job is not an error: cancel is
// idempotent, and the caller's intent is satisfied either way.
}).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const describeImport: DataImportServiceContract["describeImport"] = (input) =>
Effect.gen(function* () {
const job = yield* fromStore(
"findJob",
repository.findJob({
id: input.id,
organizationId: input.organizationId,
})
);
if (Option.isNone(job)) {
return yield* new DataImportNotFoundError({
message: "No import with this id exists in this workspace.",
});
}
const window = clampReportWindow(input.limit, input.offset);
// The row window, its summary, and the pending board names are three
// reads of the same staged job and none needs another's answer, so they
// resolve together; the report is polled while an import runs, which is
// exactly when it should not cost three round trips.
const [report, summary, newBoards] = yield* Effect.all(
[
fromStore(
"listRows",
repository.listRows({
board: input.board,
filter: input.filter,
jobId: input.id,
limit: window.limit,
message: input.message,
offset: window.offset,
sort: input.sort,
})
),
fromStore(
"summarizeReport",
repository.summarizeReport({ jobId: input.id })
),
// A staged file may name boards that do not exist yet, and the
// confirmation names them. Once the job runs, the worker creates
// them, so an unchanged list would only be stale.
canConfirmDataImportStatus(job.value.status)
? fromStore(
"listNewBoardNames",
repository.listNewBoardNames({ jobId: input.id })
)
: Effect.succeed([]),
],
{ concurrency: "unbounded" }
);
return {
job: job.value,
newBoards: [...newBoards],
report: {
boardGroups: summary.boardGroups,
counts: summary.counts,
failureGroups: summary.failureGroups,
rows: report.rows,
total: report.total,
},
};
}).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const listImports: DataImportServiceContract["listImports"] = (input) =>
fromStore(
"listJobs",
repository.listJobs({
limit: IMPORT_JOB_LIST_LIMIT,
organizationId: input.organizationId,
})
).pipe(Policy.withPolicy(canImportPosts(input.organizationId)));
const watchImport: DataImportServiceContract["watchImport"] = (input) =>
Stream.fromEffectSchedule(
suppressInternalFailures(describeImport(input)),
Schedule.spaced(DATA_IMPORT_WATCH_MS)
).pipe(
Stream.filterMap(
Filter.fromPredicateOption(
(option: Option.Option) => option
)
),
// A job still moves through the active statuses on its own; everything
// else is final. This watch writes its last snapshot and ends on the
// transition out — the predicate is the machine's, the same graph every
// guarded `UPDATE` in the repository asks.
Stream.takeUntil(
(detail) => !isActiveDataImportStatus(detail.job.status)
),
excludeUnchanged(reportChangeKey)
);
const watchImportList: DataImportServiceContract["watchImportList"] = (
input
) =>
Stream.fromEffectSchedule(
suppressInternalFailures(
listImports({ organizationId: input.organizationId })
),
Schedule.spaced(DATA_IMPORT_WATCH_MS)
).pipe(
Stream.filterMap(
Filter.fromPredicateOption(
(option: Option.Option) => option
)
),
excludeUnchanged((jobs) => jobs.map(jobChangeKey).join("\u0001"))
);
return {
cancelImport,
confirmImport,
describeImport,
listImports,
listProviders,
stageBoardImport,
stageProviderImport,
watchImport,
watchImportList,
} satisfies DataImportServiceContract;
});
export class DataImportService extends Context.Service<
DataImportService,
DataImportServiceContract
>()("DataImportService", {
make: makeDataImportService,
}) {
static readonly layer = Layer.effect(this, this.make).pipe(
Layer.provide(DataTransferRepository.layer),
Layer.provide(BoardRepository.layer),
Layer.provide(PostStatusRepository.layer)
);
}