import * as NodeUtil from "node:util"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { Command, Flag, GlobalFlag, Prompt } from "effect/cli"; import { FetchHttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; import * as BootService from "../cloud/bootService.ts"; import * as ServiceProtection from "../cloud/serviceProtection.ts"; import { compareExactServiceVersions, SERVICE_ENV_FILE } from "../cloud/serviceProtocol.ts"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; import { resolveEntryPath, resolveHostCliHint } from "./invocation.ts"; export const layer = (config: ServerConfig.ServerConfig["Service"]) => BootService.layer({ baseDir: config.baseDir, logsDir: config.logsDir, cliVersion: packageJson.version, }).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(FetchHttpClient.layer)); export type ServiceReconcileResult = | { readonly changed: false; readonly status: BootService.BootServiceStatus; } | { readonly changed: true; readonly previouslyInstalled: boolean; readonly plan: BootService.BootServicePlan; }; const SERVICE_ENV_KEY = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/u; /** * Sets or removes (`undefined`) entries in `service.env` text and keeps every * other line, including comments, as the user wrote it. */ export const upsertServiceEnv = ( contents: string, entries: Readonly>, ): string => { const written = new Set(); const source = contents.split("\n"); const lines: string[] = []; for (let index = 0, offset = 0; index < source.length;) { const line = source[index]!; const match = SERVICE_ENV_KEY.exec(line); let end = index + 1; if (match !== null) { // util.parseEnv lets a quoted value run to its closing quote across lines, // skipping only spaces, tabs and CRs before the opening quote. const rest = line.slice(match[0].length).replace(/^[ \t\r]+/u, ""); const quote = rest[0]; if ((quote === '"' || quote === "'" || quote === "`") && !rest.includes(quote, 1)) { const closing = contents.indexOf(quote, offset + line.length); if (closing !== -1) { end += contents.slice(offset + line.length, closing).split("\n").length - 1; } } } const span = source.slice(index, end); offset += span.reduce((length, part) => length + part.length + 1, 0); index = end; const key = match?.[1]; if (key === undefined || !Object.hasOwn(entries, key)) { lines.push(span.join("\n")); continue; } const value = entries[key]; if (value === undefined || written.has(key)) continue; written.add(key); lines.push(`${key}=${value}`); } while (lines.at(-1) === "") lines.pop(); for (const [key, value] of Object.entries(entries)) { if (value !== undefined && !written.has(key)) lines.push(`${key}=${value}`); } return lines.length === 0 ? "" : `${lines.join("\n")}\n`; }; /** `service.env` under `baseDir`, parsed as the service launcher parses it. */ export const readServiceEnv = Effect.fn("cli.service.read_env")(function* (baseDir: string) { const fs = yield* FileSystem.FileSystem; const path = (yield* Path.Path).join(baseDir, SERVICE_ENV_FILE); const contents = (yield* fs.exists(path)) ? yield* fs.readFileString(path) : ""; const entries = Object.entries(NodeUtil.parseEnv(contents)).flatMap(([key, value]) => value === undefined ? [] : [[key, value] as const], ); return { path, contents, entries: Object.fromEntries(entries) }; }); export const writeServiceEnv = Effect.fn("cli.service.write_env")(function* ( baseDir: string, entries: Readonly>, ) { const fs = yield* FileSystem.FileSystem; const current = yield* readServiceEnv(baseDir); const next = upsertServiceEnv(current.contents, entries); if (next === current.contents) return; yield* fs.makeDirectory(baseDir, { recursive: true }); yield* fs.writeFileString(current.path, next); }); export const reconcileService = Effect.fn("cli.service.reconcile")(function* (options?: { readonly allowDowngrade?: boolean; readonly start?: boolean; }) { const { baseDir } = yield* ServerConfig.ServerConfig; const service = yield* BootService.BootService; const status = yield* service.status; // Before the downgrade check, which reads this home's state, not the service's. if (status.otherBaseDir !== undefined) { return yield* new BootService.BootServiceOtherBaseDirError({ installedBaseDir: status.otherBaseDir, baseDir, }); } if (status.installed && status.current) { return { changed: false, status } satisfies ServiceReconcileResult; } if (status.supported && !status.installed) { // A new service listens on this machine only, until `t3 up` opens the local network. const { entries } = yield* readServiceEnv(baseDir); if (entries.T3CODE_LAN === undefined && entries.T3CODE_HOST === undefined) { yield* writeServiceEnv(baseDir, { T3CODE_LAN: "false" }); } } if ( status.installedVersion !== undefined && options?.allowDowngrade !== true && compareExactServiceVersions(packageJson.version, status.installedVersion) < 0 ) { return yield* new BootService.BootServiceDowngradeRefusedError({ installedVersion: status.installedVersion, targetVersion: packageJson.version, }); } const plan = yield* service.install(options); return { changed: true, previouslyInstalled: status.installed, plan, } satisfies ServiceReconcileResult; }); export function formatServiceChange( result: Extract, ): string { if (result.plan.handedOff === true) { return `Service update to t3@${packageJson.version} queued. This session will disconnect.\nLog: ${result.plan.logPath}`; } return `Service ${result.previouslyInstalled ? "updated" : "installed"} · t3@${packageJson.version}\nLog: ${result.plan.logPath}`; } function formatProtection( protection: ServiceProtection.ServiceProtectionState | undefined, ): ReadonlyArray { if (protection === undefined) return []; return [ protection.unprotected === undefined ? ` Memory protection: on${protection.setByProtect ? " (t3 service protect)" : ""}` : ` Memory protection: off, ${protection.unprotected} grants none. Run \`t3 service protect\`.`, ...(protection.oomScoreAdj === undefined ? [] : [` OOM score: ${protection.oomScoreAdj}`]), ]; } function formatServiceStatus( status: BootService.BootServiceStatus, cliVersion: string, commands: { readonly install: string; readonly update: string }, protection?: ServiceProtection.ServiceProtectionState, ): string { if (!status.supported) { return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd, Windows with Task Scheduler"; } if (!status.installed) { return `T3 Code service\n Status: not installed\n Next: Run \`${commands.install}\`.`; } if (status.otherBaseDir !== undefined) { return [ "T3 Code service", " Status: installed for another T3 home", ` Base dir: ${status.otherBaseDir}`, ` Next: Pass \`${BootService.formatBootServiceBaseDirOption(status.otherBaseDir)}\` to manage it.`, ].join("\n"); } const installedVersion = status.installedVersion ?? cliVersion; const problems = (status.problems ?? []).map( (problem) => ` [${problem}] ${BootService.formatBootServiceProblem(problem)}`, ); if ( !status.current && status.installedVersion !== undefined && compareExactServiceVersions(status.installedVersion, cliVersion) > 0 ) { return [ "T3 Code service", ` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, ...formatProtection(protection), ...problems, ` Next: Run \`${commands.update}\` to match it, or pass \`--allow-downgrade\` to \`${commands.install}\` explicitly.`, ].join("\n"); } return [ "T3 Code service", ` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, ...formatProtection(protection), ...problems, ...(status.current ? [] : [` Next: Run \`${commands.install}\` to repair it.`]), ].join("\n"); } const runServiceCommand = Effect.fn("cli.service.run")(function* ( flags: { readonly baseDir: Parameters[0]["baseDir"] }, run: Effect.Effect, ) { const logLevel = yield* GlobalFlag.LogLevel; const config = yield* resolveCliAuthConfig(flags, logLevel); return yield* run.pipe(Effect.provide(Layer.merge(layer(config), ServerConfig.layer(config)))); }); const serviceReconcileFlags = { ...projectLocationFlags, allowDowngrade: Flag.Boolean("allow-downgrade").pipe( Flag.withDescription("Allow replacing a newer installed service with this older CLI version."), Flag.withDefault(false), ), }; const serviceInstallCommand = Command.make("install", serviceReconcileFlags).pipe( Command.withDescription( "Install T3 Code as a background service for this user. `t3 up` also chooses how your other devices reach it.", ), Command.withHandler((flags) => runServiceCommand( flags, Effect.gen(function* () { const result = yield* reconcileService({ allowDowngrade: flags.allowDowngrade }); if (!result.changed) { yield* Console.log( `T3 Code service is already installed with t3@${packageJson.version} (${yield* resolveEntryPath}).`, ); return; } yield* Console.log(formatServiceChange(result)); if (!result.previouslyInstalled) { const upCommand = yield* resolveHostCliHint("up", flags.baseDir); yield* Console.log(`Run \`${upCommand}\` to choose how your other devices reach it.`); } }), ), ), ); const serviceUpdateCommand = Command.make("update", serviceReconcileFlags).pipe( Command.withDescription("Deprecated. Run `t3 update` to move to a newer release."), Command.unlisted, Command.withHandler((flags) => runServiceCommand( flags, Effect.gen(function* () { yield* Console.log( "`t3 service update` is deprecated: run `t3 update` to move to a newer release, or `t3 service install` to repair the service. Repairing now.", ); const result = yield* reconcileService({ allowDowngrade: flags.allowDowngrade }); if (!result.changed) { yield* Console.log( `T3 Code service is already using t3@${packageJson.version} (${yield* resolveEntryPath}).`, ); return; } yield* Console.log(formatServiceChange(result)); }), ), ), ); const serviceRestartCommand = Command.make("restart", projectLocationFlags).pipe( Command.withDescription( "Restart the background service. Picks up a version installed by `t3 update` that was not restarted at the time.", ), Command.withHandler((flags) => runServiceCommand( flags, Effect.gen(function* () { const service = yield* BootService.BootService; const status = yield* service.status; const outcome = yield* service.restart; const onVersion = status.installedVersion === undefined ? "" : ` on t3@${status.installedVersion}`; yield* Console.log( outcome === "restarted" ? `Restarted the T3 Code service${onVersion}.` : outcome === "queued" ? `Queued a restart of the T3 Code service${onVersion}. This session runs inside the service and disconnects when it goes down.\nLogs: ${status.logPath}` : status.otherBaseDir !== undefined ? `The T3 Code service serves ${status.otherBaseDir}, so it was not restarted. Pass \`${BootService.formatBootServiceBaseDirOption(status.otherBaseDir)}\` to restart it.` : "T3 Code service is not installed.", ); }), ), ), ); const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags).pipe( Command.withDescription("Stop and remove the T3 Code background service."), Command.withHandler((flags) => runServiceCommand( flags, Effect.gen(function* () { const service = yield* BootService.BootService; const path = yield* Path.Path; const { baseDir } = yield* ServerConfig.ServerConfig; // An unreadable unit stays removable; only a unit known to serve another home is kept. const installedBaseDir = yield* service.status.pipe( Effect.map((status) => status.installedBaseDir), Effect.orElseSucceed(() => undefined), ); if ( installedBaseDir !== undefined && path.resolve(installedBaseDir) !== path.resolve(baseDir) ) { yield* Console.log( `The T3 Code service serves ${installedBaseDir}, so it was not removed. Pass \`${BootService.formatBootServiceBaseDirOption(installedBaseDir)}\` to remove it.`, ); return; } const removed = yield* service.uninstall; yield* Console.log( removed ? "Removed the T3 Code service." : "T3 Code service is not installed.", ); }), ), ), ); const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe( Command.withDescription("Show whether the T3 Code background service is installed."), Command.withHandler((flags) => runServiceCommand( flags, Effect.gen(function* () { const service = yield* BootService.BootService; const path = yield* Path.Path; const status = yield* service.status; const protection = status.installed && (yield* HostProcess.Platform) === "linux" ? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe( Effect.provide(ProcessRunner.layer), ) : undefined; const commands = { install: yield* resolveHostCliHint("service install", flags.baseDir), update: yield* resolveHostCliHint("update", flags.baseDir, [ status.installedVersion ?? packageJson.version, ]), }; yield* Console.log(formatServiceStatus(status, packageJson.version, commands, protection)); }), ), ), ); const formatDropIns = (dropIns: ReadonlyArray) => dropIns.map( ({ path, contents }) => ` ${path}: ${contents.trimEnd().split("\n").slice(1).join(" ")}`, ); const confirm = (yes: boolean, message: string) => yes ? Effect.succeed(true) : Prompt.run(Prompt.Confirm({ message, initial: false })); const serviceProtectCommand = Command.make("protect", { ...projectLocationFlags, undo: Flag.Boolean("undo").pipe( Flag.withDescription("Remove the protection this command set up."), Flag.withDefault(false), ), yes: Flag.Boolean("yes").pipe( Flag.withDescription("Answer yes to every question, including SSH and Tailscale."), Flag.withDefault(false), ), }).pipe( Command.withDescription( "Protect the T3 Code server from agents under memory pressure (Linux with systemd). Asks for sudo.", ), Command.withHandler((flags) => Effect.gen(function* () { const unsupported = "Service protection needs Linux with systemd."; if ((yield* HostProcess.Platform) !== "linux") return yield* Console.log(unsupported); // Before resolving config, which creates T3 directories that root would then own. if ((yield* HostProcess.UserId) === 0) { return yield* Console.log( "Run `t3 service protect` as the user that runs T3 Code, without sudo. It asks for sudo when it needs root.", ); } return yield* runServiceCommand( flags, Effect.gen(function* () { const uid = yield* HostProcess.UserId; const remote = yield* ServiceProtection.detectRemoteAccessUnits; const { unitPath } = yield* (yield* BootService.BootService).status; if (uid === undefined || remote === undefined || unitPath === "") { return yield* Console.log(unsupported); } const userUnitDir = (yield* Path.Path).dirname(unitPath); if (flags.undo) { const installed = yield* ServiceProtection.installedProtection( ServiceProtection.protectionPlan({ uid, userUnitDir, remoteAccessUnits: ServiceProtection.REMOTE_ACCESS_UNITS, }), ); if (installed === undefined) { return yield* Console.log("T3 Code service protection is not set up."); } yield* Console.log( [ "Removes:", ...[...installed.system, ...installed.user, installed.managerDefaults].map( ({ path, shared }) => ` ${path}${shared === true ? " (kept while another user is protected)" : ""}`, ), ].join("\n"), ); if (!(yield* confirm(flags.yes, "Remove T3 Code service protection?"))) return; yield* ServiceProtection.removeProtection(installed); return yield* Console.log( "Removed T3 Code service protection. The server's OOM score returns to normal after `t3 service restart`.", ); } yield* Console.log( [ "Keeps T3 Code running when agents fill this machine's memory:", " - reserves memory for the T3 Code server, so the kernel takes memory from agents first", " - makes the out-of-memory killer pick agents before the server", "It adds systemd drop-in files and changes nothing else. Undo with `t3 service protect --undo`.", ].join("\n"), ); const names = remote.map((unit) => unit.replace(/\.service$/, "")).join(" and "); const protectRemote = remote.length > 0 && (yield* confirm( flags.yes, `Also protect ${names} so you can still reach this machine under memory pressure?`, )); const plan = ServiceProtection.protectionPlan({ uid, userUnitDir, remoteAccessUnits: protectRemote ? remote : [], }); yield* Console.log( [ "Drop-ins it writes, as root through sudo:", ...formatDropIns(plan.system), "As you:", ...formatDropIns([plan.managerDefaults, ...plan.user]), ].join("\n"), ); if (!(yield* confirm(flags.yes, "Apply these settings?"))) return; yield* ServiceProtection.applyProtection(plan); yield* Console.log( "Protected T3 Code. Memory protection applies now; the server takes its OOM score after `t3 service restart`. Undo with `t3 service protect --undo`.", ); }).pipe(Effect.provide(ProcessRunner.layer)), ); }), ), ); export const serviceCommand = Command.make("service").pipe( Command.withDescription("Manage the T3 Code background service."), Command.withSubcommands([ serviceInstallCommand, serviceRestartCommand, serviceUninstallCommand, serviceStatusCommand, serviceProtectCommand, serviceUpdateCommand, ]), );