// /////////////////////////////////////////////////////////////////////////// // // # ACE - Active Consistency Engine // // Copyright (C) 2023 - 2026, pgEdge (https://www.pgedge.com/) // // This software is released under the PostgreSQL License: // https://opensource.org/license/postgresql // // /////////////////////////////////////////////////////////////////////////// package cli import ( "context" _ "embed" "errors" "fmt" "os" "os/signal" "path/filepath" "strconv" "strings" "sync" "syscall" "github.com/charmbracelet/log" "github.com/google/uuid" "github.com/pgedge/ace/internal/api/http" "github.com/pgedge/ace/internal/consistency/diff" "github.com/pgedge/ace/internal/consistency/mtree" "github.com/pgedge/ace/internal/consistency/repair" "github.com/pgedge/ace/internal/infra/cdc" "github.com/pgedge/ace/internal/jobs" utils "github.com/pgedge/ace/pkg/common" "github.com/pgedge/ace/pkg/config" "github.com/pgedge/ace/pkg/logger" "github.com/urfave/cli/v3" ) //go:embed default_config.yaml var defaultConfigYAML string //go:embed default_pg_service.conf var defaultPgServiceConf string // SetupCLI builds the ace command with all its subcommands and flags. func SetupCLI(version string) *cli.Command { // Use -V for version, so -v stays reserved for the debug/verbose flag on // subcommands, matching go tooling conventions. cli.VersionFlag = &cli.BoolFlag{ Name: "version", Aliases: []string{"V"}, Usage: "print the version", } commonFlags := []cli.Flag{ &cli.StringFlag{ Name: "dbname", Aliases: []string{"d"}, Usage: "Name of the database", Value: "", }, &cli.StringFlag{ Name: "nodes", Aliases: []string{"n"}, Usage: "Nodes to include in the diff (default: all)", Value: "all", }, &cli.BoolFlag{ Name: "quiet", Aliases: []string{"q"}, Usage: "Whether to suppress output", Value: false, }, &cli.BoolFlag{ Name: "debug", Aliases: []string{"v"}, Usage: "Enable debug logging", Value: false, }, } runTimeoutFlag := &cli.IntFlag{ Name: "run-timeout", Usage: "Seconds for the diff of one table, 0 = no limit (default: table_diff.run_timeout, or 0)", } diffFlags := []cli.Flag{ &cli.StringFlag{ Name: "block-size", Aliases: []string{"b"}, Usage: "Number of rows per block", Value: "100000", }, &cli.Float64Flag{ Name: "concurrency-factor", Aliases: []string{"c"}, Usage: "CPU ratio for the comparison of the blocks that differ (0.0–4.0, e.g. 0.5 uses half of available CPUs); hashing uses --max-connections", Value: 0.5, }, &cli.IntFlag{ Name: "compare-unit-size", Aliases: []string{"u"}, Usage: "Deprecated: no longer used and ignored", Value: 0, }, &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, Usage: "Output format", Value: "json", }, &cli.BoolFlag{ Name: "override-block-size", Aliases: []string{"B"}, Usage: "Override block size", Value: false, }, &cli.IntFlag{ Name: "max-connections", Aliases: []string{"M"}, Usage: "Connections to each node for one table-diff task; a set value is a strict limit for the whole task (0 = max_connections from the config file; if that is not set, 8 for cutting and hashing, and the comparison can open more)", Value: 0, }, &cli.IntFlag{ Name: "block-timeout", Usage: "Seconds for one query on one block, 0 = no limit (default: table_diff.block_timeout, or 60)", }, runTimeoutFlag, &cli.StringFlag{ Name: "slicer", Usage: "How to cut a table into blocks: auto, cutter or sample (default: table_diff.slicer from ace.yaml, or auto)", }, } rerunOnlyFlags := []cli.Flag{ &cli.StringFlag{ Name: "diff-file", Aliases: []string{"f"}, Usage: "Path to the diff result directory, or to the JSON report next to it, to rerun from (required)", Required: true, }, } skipFlags := []cli.Flag{ &cli.StringFlag{ Name: "skip-tables", Aliases: []string{"T"}, Usage: "Comma-separated list of tables to skip", }, &cli.StringFlag{ Name: "skip-file", Aliases: []string{"s"}, Usage: "Path to a file with a list of tables to skip", }, } tableDiffFlags := append(commonFlags, diffFlags...) tableDiffFlags = append(tableDiffFlags, &cli.Int64Flag{ Name: "max-html-rows", Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)", Value: 0, }, &cli.Int64Flag{ Name: "max-html-value-length", Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)", Value: 0, }, &cli.StringFlag{ Name: "table-filter", Aliases: []string{"F"}, Usage: "Where clause expression to use while diffing tables", Value: "", }, &cli.StringFlag{ Name: "against-origin", Usage: "Restrict diff to rows whose node_origin matches this Spock node id or name", Value: "", }, &cli.StringFlag{ Name: "until", Usage: "Optional commit timestamp upper bound (RFC3339) for rows to include", Value: "", }, // The block hash no longer uses pgcrypto. The flag is kept, hidden, // so that scripts that still pass it do not fail. &cli.BoolFlag{ Name: "ensure-pgcrypto", Usage: "Deprecated: has no effect; table-diff no longer needs pgcrypto", Value: false, Hidden: true, }, &cli.BoolFlag{ Name: "schedule", Aliases: []string{"S"}, Usage: "Schedule a table-diff job to run periodically", Value: false, }, &cli.StringFlag{ Name: "every", Aliases: []string{"e"}, Usage: "Time duration (e.g., 5m, 3h, etc.)", Value: "", }, ) diffReportFlags := []cli.Flag{ &cli.StringFlag{ Name: "diff-file", Aliases: []string{"f"}, Usage: "Path to the diff result directory, or to the JSON report next to it (required)", Required: true, }, &cli.Int64Flag{ Name: "max-html-rows", Usage: "Max rows per node pair in the HTML report (0 = use table_diff.max_html_rows from config, default 10000)", Value: 0, }, &cli.Int64Flag{ Name: "max-html-value-length", Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)", Value: 0, }, &cli.BoolFlag{ Name: "quiet", Aliases: []string{"q"}, Usage: "Whether to suppress output", Value: false, }, &cli.BoolFlag{ Name: "debug", Aliases: []string{"v"}, Usage: "Enable debug logging", Value: false, }, } tableRerunFlags := append(commonFlags, rerunOnlyFlags...) tableRerunFlags = append(tableRerunFlags, runTimeoutFlag) tableRepairFlags := []cli.Flag{ &cli.StringFlag{ Name: "diff-file", Aliases: []string{"f"}, Usage: "Path to the diff result directory, or to the JSON report next to it (required)", Required: true, }, &cli.StringFlag{ Name: "repair-plan", Aliases: []string{"p"}, Usage: "Path to the advanced repair plan (YAML/JSON); skips source-of-truth requirement", }, &cli.StringFlag{ Name: "source-of-truth", Aliases: []string{"r"}, Usage: "Name of the node to be considered the source of truth", }, &cli.BoolFlag{ Name: "dry-run", Aliases: []string{"y"}, Usage: "Show what would be done without executing", Value: false, }, &cli.BoolFlag{ Name: "generate-report", Aliases: []string{"g"}, Usage: "Generate a report of the repair operation", Value: false, }, &cli.BoolFlag{ Name: "insert-only", Aliases: []string{"i"}, Usage: "Only perform inserts, no updates or deletes", Value: false, }, &cli.BoolFlag{ Name: "upsert-only", Aliases: []string{"P"}, Usage: "Only perform upserts (insert or update), no deletes", Value: false, }, &cli.BoolFlag{ Name: "fire-triggers", Aliases: []string{"t"}, Usage: "Whether to fire triggers during repairs", Value: false, }, &cli.BoolFlag{ Name: "bidirectional", Aliases: []string{"Z"}, Usage: "Whether to perform repairs in both directions. Can be used only with the insert-only option", Value: false, }, &cli.BoolFlag{ Name: "recovery-mode", Usage: "Enable recovery-mode repair using origin-only diffs", Value: false, }, &cli.BoolFlag{ Name: "preserve-origin", Usage: "Preserve replication origin node ID and commit timestamp for repaired rows", Value: false, }, &cli.BoolFlag{ Name: "fix-nulls", Aliases: []string{"X"}, Usage: "Fill NULL columns on each node using non-NULL values from its peers (no source-of-truth required)", Value: false, }, } tableRepairFlags = append(commonFlags, tableRepairFlags...) spockDiffFlags := append(commonFlags, &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, Usage: "Output format", Value: "json", }) repsetDiffFlags := append(commonFlags, diffFlags...) repsetDiffFlags = append(repsetDiffFlags, skipFlags...) repsetDiffFlags = append(repsetDiffFlags, &cli.BoolFlag{ Name: "schedule", Aliases: []string{"S"}, Usage: "Schedule a repset-diff job to run periodically", Value: false, }, &cli.StringFlag{ Name: "every", Aliases: []string{"e"}, Usage: "Time duration (e.g., 5m, 3h, etc.)", Value: "", }, ) schemaDiffFlags := append(commonFlags, diffFlags...) schemaDiffFlags = append(schemaDiffFlags, skipFlags...) schemaDiffFlags = append(schemaDiffFlags, &cli.BoolFlag{ Name: "ddl-only", Aliases: []string{"L"}, Usage: "Compare only schema objects (tables, functions, etc.), not table data", Value: false, }, &cli.StringFlag{ Name: "compare", Usage: "What to compare: \"data\" (default, per-table data diff) or \"structure\" (symmetric column/key/constraint comparison, exits nonzero with a severity-coded status when something differs)", Value: diff.CompareData, }, &cli.BoolFlag{ Name: "schedule", Aliases: []string{"S"}, Usage: "Schedule a schema-diff job to run periodically", Value: false, }, &cli.StringFlag{ Name: "every", Aliases: []string{"e"}, Usage: "Time duration (e.g., 5m, 3h, etc.)", Value: "", }, ) mtreeBuildFlags := []cli.Flag{ &cli.StringFlag{ Name: "block-size", Aliases: []string{"b"}, Usage: "Rows per leaf block", Value: "10000", }, &cli.Float64Flag{ Name: "max-cpu-ratio", Aliases: []string{"m"}, Usage: "Share of this host's CPUs to use as parallel workers per node", Value: 0.5, }, &cli.IntFlag{ Name: "max-connections", Aliases: []string{"M"}, Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)", Value: 0, }, &cli.BoolFlag{ Name: "override-block-size", Aliases: []string{"B"}, Usage: "Skip block size check, and potentially tolerate unsafe block sizes", Value: false, }, &cli.BoolFlag{ Name: "analyse", Aliases: []string{"a"}, Usage: "Run ANALYZE on the table", Value: false, }, &cli.BoolFlag{ Name: "recreate-objects", Aliases: []string{"R"}, Usage: "Drop and recreate Merkle tree objects", Value: false, }, &cli.BoolFlag{ Name: "write-ranges", Aliases: []string{"w"}, Usage: "Write block ranges to a JSON file", Value: false, }, &cli.StringFlag{ Name: "ranges-file", Aliases: []string{"k"}, Usage: "Path to a file with pre-computed ranges", }, } mtreeBuildFlags = append(mtreeBuildFlags, commonFlags...) mtreeUpdateFlags := []cli.Flag{ &cli.Float64Flag{ Name: "max-cpu-ratio", Aliases: []string{"m"}, Usage: "Share of this host's CPUs to use as parallel workers per node", Value: 0.5, }, &cli.IntFlag{ Name: "max-connections", Aliases: []string{"M"}, Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)", Value: 0, }, &cli.BoolFlag{ Name: "rebalance", Aliases: []string{"l"}, Usage: "Rebalance the tree by merging small blocks", Value: false, }, &cli.BoolFlag{ Name: "skip-cdc", Aliases: []string{"U"}, Usage: "Skip CDC processing (only rehash dirty blocks)", Value: false, }, &cli.IntFlag{ Name: "cdc-timeout", Usage: "Seconds to spend draining CDC before giving up (0 = use cdc_processing_timeout / default)", Value: 0, }, } mtreeUpdateFlags = append(mtreeUpdateFlags, commonFlags...) mtreeDiffFlags := []cli.Flag{ &cli.Int64Flag{ Name: "max-html-rows", Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)", Value: 0, }, &cli.Int64Flag{ Name: "max-html-value-length", Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)", Value: 0, }, &cli.Float64Flag{ Name: "max-cpu-ratio", Aliases: []string{"m"}, Usage: "Share of this host's CPUs to use as parallel workers per node", Value: 0.5, }, &cli.IntFlag{ Name: "max-connections", Aliases: []string{"M"}, Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)", Value: 0, }, &cli.StringFlag{ Name: "until", Usage: "Optional commit timestamp upper bound (RFC3339) for rows to include", Value: "", }, &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, Usage: "Output format", Value: "json", }, &cli.BoolFlag{ Name: "skip-cdc", Aliases: []string{"U"}, Usage: "Skip CDC processing (only rehash dirty blocks and compare)", Value: false, }, &cli.IntFlag{ Name: "cdc-timeout", Usage: "Seconds to spend draining CDC before giving up (0 = use cdc_processing_timeout / default)", Value: 0, }, } mtreeDiffFlags = append(mtreeDiffFlags, commonFlags...) configInitFlags := []cli.Flag{ &cli.StringFlag{ Name: "path", Aliases: []string{"p"}, Usage: "Path to write the config file", Value: "ace.yaml", }, &cli.BoolFlag{ Name: "force", Aliases: []string{"x"}, Usage: "Overwrite the config file if it already exists", }, &cli.BoolFlag{ Name: "stdout", Aliases: []string{"z"}, Usage: "Print the config to stdout instead of writing a file", }, } clusterInitFlags := []cli.Flag{ &cli.StringFlag{ Name: "path", Aliases: []string{"p"}, Usage: "Path to write the pg service file", Value: "pg_service.conf", }, &cli.BoolFlag{ Name: "force", Aliases: []string{"x"}, Usage: "Overwrite the service file if it already exists", }, &cli.BoolFlag{ Name: "stdout", Aliases: []string{"z"}, Usage: "Print the service file to stdout instead of writing a file", }, } app := &cli.Command{ Name: "ace", Usage: "ACE - Active Consistency Engine", Version: version, Commands: []*cli.Command{ { Name: "config", Usage: "Manage ACE configuration files", Commands: []*cli.Command{ { Name: "init", Usage: "Create a default ace.yaml file", Flags: configInitFlags, Action: ConfigInitCLI, }, }, }, { Name: "start", Usage: "Start the ACE scheduler for configured jobs", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "debug", Aliases: []string{"v"}, Usage: "Enable debug logging", }, &cli.StringFlag{ Name: "component", Aliases: []string{"C"}, Usage: "Component to start: scheduler, api, or all", Value: "all", }, }, Action: StartSchedulerCLI, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "server", Usage: "Run the ACE REST API server", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "debug", Aliases: []string{"v"}, Usage: "Enable debug logging", }, }, Action: StartAPIServerCLI, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "cluster", Usage: "Manage ACE cluster service definitions", Commands: []*cli.Command{ { Name: "init", Usage: "Create a sample pg_service.conf file", Flags: clusterInitFlags, Action: ClusterInitCLI, }, }, }, { Name: "table-diff", Usage: "Compare tables between PostgreSQL databases", ArgsUsage: "[cluster] ", Description: "A tool for comparing tables between PostgreSQL databases " + "and detecting data inconsistencies", Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for table-diff: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for table-diff (usage: [cluster]
)") } return TableDiffCLI(cmd) }, Flags: tableDiffFlags, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "table-rerun", Usage: "Re-run a diff from a file to check for persistent differences", ArgsUsage: "[cluster]", Flags: tableRerunFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 1 { return fmt.Errorf("unexpected arguments for table-rerun (usage: [cluster])") } return TableRerunCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "diff-report", Usage: "Build the HTML report of a finished diff from its result directory", Description: "Reads the result directory of table-diff, mtree table-diff, repset-diff " + "or schema-diff and writes the HTML report next to it. No database connection is needed.", Flags: diffReportFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 0 { return fmt.Errorf("unexpected arguments for diff-report (usage: --diff-file )") } return DiffReportCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { switch { case cmd.Bool("debug"): logger.SetLevel(log.DebugLevel) case cmd.Bool("quiet"): logger.SetLevel(log.WarnLevel) default: logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "table-repair", Usage: "Repair table inconsistencies based on a diff file", ArgsUsage: "[cluster]
", Flags: tableRepairFlags, Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for table-repair: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for table-repair (usage: [cluster]
)") } return TableRepairCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "spock-diff", Usage: "Compare spock metadata across cluster nodes", ArgsUsage: "[cluster]", Flags: spockDiffFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 1 { return fmt.Errorf("unexpected arguments for spock-diff (usage: [cluster])") } return SpockDiffCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "schema-diff", Usage: "Compare schemas across cluster nodes", ArgsUsage: "[cluster] ", Flags: schemaDiffFlags, Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for schema-diff: needs ") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for schema-diff (usage: [cluster] )") } return SchemaDiffCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "repset-diff", Usage: "Compare replication sets across cluster nodes", ArgsUsage: "[cluster] ", Flags: repsetDiffFlags, Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for repset-diff: needs ") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for repset-diff (usage: [cluster] )") } return RepsetDiffCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "mtree", Usage: "Merkle tree operations", Commands: []*cli.Command{ { Name: "init", Usage: "Initialise Merkle tree replication for a cluster", ArgsUsage: "[cluster]", Flags: commonFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 1 { return fmt.Errorf("unexpected arguments for mtree init (usage: [cluster])") } return MtreeInitCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "listen", Usage: "Listen for changes and update Merkle trees", ArgsUsage: "[cluster]", Flags: commonFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 1 { return fmt.Errorf("unexpected arguments for mtree listen (usage: [cluster])") } return MtreeListenCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "teardown", Usage: "Teardown Merkle tree replication for a cluster", ArgsUsage: "[cluster]", Flags: commonFlags, Action: func(_ context.Context, cmd *cli.Command) error { if cmd.Args().Len() > 1 { return fmt.Errorf("unexpected arguments for mtree teardown (usage: [cluster])") } return MtreeTeardownCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "teardown-table", Usage: "Teardown Merkle tree objects for a specific table", ArgsUsage: "[cluster]
", Flags: commonFlags, Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for mtree teardown-table: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for mtree teardown-table (usage: [cluster]
)") } return MtreeTeardownTableCLI(cmd) }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "build", Usage: "Build a new Merkle tree for a table", ArgsUsage: "[cluster]
", Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for mtree build: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for mtree build (usage: [cluster]
)") } return MtreeBuildCLI(cmd) }, Flags: mtreeBuildFlags, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "update", Usage: "Update an existing Merkle tree for a table", ArgsUsage: "[cluster]
", Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for mtree update: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for mtree update (usage: [cluster]
)") } return MtreeUpdateCLI(cmd) }, Flags: mtreeUpdateFlags, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, { Name: "table-diff", Usage: "Use Merkle Trees for performing table-diff", ArgsUsage: "[cluster]
", Action: func(_ context.Context, cmd *cli.Command) error { argsLen := cmd.Args().Len() if argsLen == 0 { return fmt.Errorf("missing required argument for mtree diff: needs
") } if argsLen > 2 { return fmt.Errorf("unexpected arguments for mtree diff (usage: [cluster]
)") } return MtreeDiffCLI(cmd) }, Flags: mtreeDiffFlags, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if cmd.Bool("debug") { logger.SetLevel(log.DebugLevel) } else { logger.SetLevel(log.InfoLevel) } return ctx, nil }, }, }, }, }, } return app } func initTemplateFile(cmd *cli.Command, content string, defaultPath string, label string, perm os.FileMode) error { outputPath := cmd.String("path") if outputPath == "" { outputPath = defaultPath } if cmd.Bool("stdout") || outputPath == "-" { fmt.Println(content) return nil } if !cmd.Bool("force") { if _, err := os.Stat(outputPath); err == nil { return fmt.Errorf("%s already exists at %s (use --force to overwrite)", label, outputPath) } else if !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("unable to verify existing %s at %s: %w", label, outputPath, err) } } dir := filepath.Dir(outputPath) if dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("failed to create directory %s: %w", dir, err) } } // Restrict the file before any content reaches it. pg_service.conf holds // database credentials, and with --force over an existing world-readable // copy, writing first and chmod'ing after would leave them exposed in // between. Opening without O_TRUNC lets us fix the mode, then empty the // file, then write. The Chmod is not redundant: O_CREATE's mode is // umask-masked, and ignored outright for a file that already exists. f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE, perm) if err != nil { return fmt.Errorf("failed to create %s at %s: %w", label, outputPath, err) } if err := f.Chmod(perm); err != nil { f.Close() return fmt.Errorf("failed to set permissions on %s: %w", outputPath, err) } if err := f.Truncate(0); err != nil { f.Close() return fmt.Errorf("failed to truncate %s: %w", outputPath, err) } if _, err := f.WriteString(content); err != nil { f.Close() return fmt.Errorf("failed to write %s to %s: %w", label, outputPath, err) } if err := f.Close(); err != nil { return fmt.Errorf("failed to write %s to %s: %w", label, outputPath, err) } fmt.Printf("Wrote %s to %s\n", label, outputPath) return nil } func ConfigInitCLI(_ context.Context, cmd *cli.Command) error { return initTemplateFile(cmd, defaultConfigYAML, "ace.yaml", "config file", 0o644) } func ClusterInitCLI(_ context.Context, cmd *cli.Command) error { return initTemplateFile(cmd, defaultPgServiceConf, "pg_service.conf", "pg service file", 0o600) } func resolveClusterArg(cmd, missingUsage, argsUsage string, required int, args []string) (string, []string, error) { if len(args) < required { if required == 1 { return "", nil, fmt.Errorf("missing required argument for %s: needs %s", cmd, missingUsage) } return "", nil, fmt.Errorf("missing required arguments for %s: needs %s", cmd, missingUsage) } if len(args) == required { cluster := config.DefaultCluster() if cluster == "" { if required > 0 { // The positional(s) supplied were consumed as the required entity // (e.g. ), leaving the cluster unset. Spell that out so a // lone argument isn't misread as the cluster. return "", nil, fmt.Errorf( "cluster name is required: %q was read as the %s argument (usage: %s %s); "+ "pass the cluster as the first argument or set default_cluster in ace.yaml", strings.Join(args, " "), missingUsage, cmd, argsUsage) } return "", nil, fmt.Errorf("cluster name is required: specify one explicitly or set default_cluster in ace.yaml") } return cluster, args, nil } if len(args) == required+1 { cluster := strings.TrimSpace(args[0]) if cluster == "" { return "", nil, fmt.Errorf("cluster name is required: specify one explicitly or set default_cluster in ace.yaml") } return cluster, args[1:], nil } return "", nil, fmt.Errorf("unexpected arguments for %s (usage: %s)", cmd, argsUsage) } // timeoutFlag returns the value of a timeout flag in seconds, or nil if the // flag is not given, so that the value from ace.yaml is used. func timeoutFlag(cmd *cli.Command, name string) (*int, error) { if !cmd.IsSet(name) { return nil, nil } v := cmd.Int(name) if v < 0 { return nil, fmt.Errorf("--%s must be >= 0, got %d", name, v) } return &v, nil } // TableDiffCLI runs the table-diff command once, or on a schedule with // --schedule. In a single run, SIGINT or SIGTERM stops it, and the error // then names the signal. With --schedule, a signal ends the schedule: a run // in progress fails with the signal in its logged error, and the command // ends without an error. func TableDiffCLI(cmd *cli.Command) (err error) { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("table-diff", "
", "[cluster]
", 1, args) if err != nil { return err } blockSizeStr := cmd.String("block-size") blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64) if err != nil { return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err) } task := diff.NewTableDiffTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DBName = cmd.String("dbname") task.BlockSize = int(blockSizeInt) task.ConcurrencyFactor = cmd.Float64("concurrency-factor") task.MaxConnections = cmd.Int("max-connections") if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil { return err } if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil { return err } task.Slicer = cmd.String("slicer") task.CompareUnitSize = cmd.Int("compare-unit-size") task.Output = strings.ToLower(cmd.String("output")) task.MaxHTMLRows = cmd.Int64("max-html-rows") task.MaxHTMLValueLength = cmd.Int64("max-html-value-length") task.Nodes = cmd.String("nodes") if cmd.Bool("ensure-pgcrypto") { logger.Warn("--ensure-pgcrypto has no effect: table-diff no longer needs pgcrypto") } scheduleEnabled := cmd.Bool("schedule") scheduleEvery := cmd.String("every") task.TableFilter = cmd.String("table-filter") task.AgainstOrigin = cmd.String("against-origin") task.Until = cmd.String("until") task.QuietMode = cmd.Bool("quiet") task.OverrideBlockSize = cmd.Bool("override-block-size") ctx, stop := signalContext(context.Background()) defer stop() defer func() { err = withStopCause(ctx, err) }() task.Ctx = ctx if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if !scheduleEnabled { if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.ExecuteTask(); err != nil { return fmt.Errorf("error during comparison: %w", err) } return nil } freq, err := scheduler.ParseFrequency(scheduleEvery) if err != nil { return err } job := scheduler.Job{ Name: fmt.Sprintf("table-diff:%s", task.QualifiedTableName), Frequency: freq, RunOnStart: true, Task: func(runCtx context.Context) error { runTask := task.CloneForSchedule(runCtx) if err := runTask.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := runTask.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := runTask.ExecuteTask(); err != nil { return fmt.Errorf("error during comparison: %w", err) } return nil }, } return scheduler.RunSingleJob(ctx, job) } func MtreeInitCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, _, err := resolveClusterArg("mtree init", "", "[cluster]", 0, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.Mode = "init" task.Ctx = context.Background() if err := task.MtreeInit(); err != nil { return fmt.Errorf("error during mtree init: %w", err) } return nil } func MtreeListenCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, _, err := resolveClusterArg("mtree listen", "", "[cluster]", 0, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.Mode = "listen" task.Ctx = context.Background() if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } appCtx, cancel := context.WithCancel(context.Background()) defer cancel() // Handle graceful shutdown sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) go func() { <-sigChan logger.Info("Shutdown signal received, stopping listeners...") cancel() }() var wg sync.WaitGroup clusterNodes := task.GetClusterNodes() wg.Add(len(clusterNodes)) for _, nodeInfo := range clusterNodes { go func(ni map[string]any) { defer wg.Done() cdc.ListenForChanges(appCtx, ni) }(nodeInfo) } logger.Info("CDC listeners started. Press Ctrl+C to stop.") wg.Wait() logger.Info("All CDC listeners stopped.") return nil } func MtreeTeardownCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, _, err := resolveClusterArg("mtree teardown", "", "[cluster]", 0, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.Mode = "teardown" task.Ctx = context.Background() if err := task.MtreeTeardown(); err != nil { return fmt.Errorf("error during mtree teardown: %w", err) } return nil } func MtreeTeardownTableCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("mtree teardown-table", "
", "[cluster]
", 1, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.Mode = "teardown-table" task.Ctx = context.Background() if err := task.MtreeTeardownTable(); err != nil { return fmt.Errorf("error during mtree table teardown: %w", err) } return nil } func MtreeBuildCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("mtree build", "
", "[cluster]
", 1, args) if err != nil { return err } blockSizeStr := cmd.String("block-size") blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64) if err != nil { return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err) } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.BlockSize = int(blockSizeInt) task.MaxCpuRatio = cmd.Float64("max-cpu-ratio") task.MaxConnections = cmd.Int("max-connections") task.OverrideBlockSize = cmd.Bool("override-block-size") task.Analyse = cmd.Bool("analyse") task.RecreateObjects = cmd.Bool("recreate-objects") task.WriteRanges = cmd.Bool("write-ranges") task.RangesFile = cmd.String("ranges-file") task.Mode = "build" task.Ctx = context.Background() if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.BuildMtree(); err != nil { return fmt.Errorf("error during merkle tree build: %w", err) } return nil } func MtreeUpdateCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("mtree update", "
", "[cluster]
", 1, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.MaxCpuRatio = cmd.Float64("max-cpu-ratio") task.MaxConnections = cmd.Int("max-connections") task.Rebalance = cmd.Bool("rebalance") task.NoCDC = cmd.Bool("skip-cdc") task.CDCTimeoutSec = cmd.Int("cdc-timeout") task.Mode = "update" task.Ctx = context.Background() if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := task.RunChecks(true /* skipValidation */); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.UpdateMtree(true /* skipAllChecks */); err != nil { return fmt.Errorf("error during merkle tree update: %w", err) } return nil } func MtreeDiffCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("mtree diff", "
", "[cluster]
", 1, args) if err != nil { return err } task := mtree.NewMerkleTreeTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") task.MaxCpuRatio = cmd.Float64("max-cpu-ratio") task.MaxConnections = cmd.Int("max-connections") task.Output = cmd.String("output") task.MaxHTMLRows = cmd.Int64("max-html-rows") task.MaxHTMLValueLength = cmd.Int64("max-html-value-length") task.NoCDC = cmd.Bool("skip-cdc") task.CDCTimeoutSec = cmd.Int("cdc-timeout") task.Until = cmd.String("until") task.Mode = "diff" task.Ctx = context.Background() if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.DiffMtree(); err != nil { return fmt.Errorf("error during merkle tree diff: %w", err) } return nil } // TableRerunCLI compares again the rows listed in a diff file. SIGINT or // SIGTERM stops it, and the error then names the signal. func TableRerunCLI(cmd *cli.Command) (err error) { args := cmd.Args().Slice() clusterName, _, err := resolveClusterArg("table-rerun", "", "[cluster]", 0, args) if err != nil { return err } task := diff.NewTableDiffTask() task.TaskID = uuid.NewString() task.Mode = "rerun" task.ClusterName = clusterName task.DiffFilePath = cmd.String("diff-file") task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.QuietMode = cmd.Bool("quiet") if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil { return err } ctx, stop := signalContext(context.Background()) defer stop() defer func() { err = withStopCause(ctx, err) }() task.Ctx = ctx if err := task.ExecuteTask(); err != nil { return fmt.Errorf("error during table-rerun: %w", err) } return nil } // DiffReportCLI builds the HTML report of a finished diff from its result // directory. A limit of 0 is taken from the table_diff section of the config // file, and then from the defaults. func DiffReportCLI(cmd *cli.Command) error { maxRows := cmd.Int64("max-html-rows") maxLen := cmd.Int64("max-html-value-length") if maxRows < 0 { return fmt.Errorf("--max-html-rows must be >= 0") } if maxLen < 0 { return fmt.Errorf("--max-html-value-length must be >= 0") } if cfg := config.Get(); cfg != nil { if cfg.TableDiff.MaxHTMLRows < 0 { return fmt.Errorf("table_diff.max_html_rows in the config must be >= 0, got %d", cfg.TableDiff.MaxHTMLRows) } if cfg.TableDiff.MaxHTMLValueLength < 0 { return fmt.Errorf("table_diff.max_html_value_length in the config must be >= 0, got %d", cfg.TableDiff.MaxHTMLValueLength) } if maxRows == 0 && cfg.TableDiff.MaxHTMLRows > 0 { maxRows = cfg.TableDiff.MaxHTMLRows } if maxLen == 0 && cfg.TableDiff.MaxHTMLValueLength > 0 { maxLen = cfg.TableDiff.MaxHTMLValueLength } } r, err := utils.OpenDiffResult(cmd.String("diff-file")) if err != nil { return err } ctx, stop := signalContext(context.Background()) defer stop() path, err := utils.WriteHTMLDiffReport(ctx, r, maxRows, maxLen) if err != nil { return withStopCause(ctx, err) } logger.Info("HTML diff report written to %s", path) return nil } func TableRepairCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("table-repair", "
", "[cluster]
", 1, args) if err != nil { return err } task := repair.NewTableRepairTask() task.ClusterName = clusterName task.QualifiedTableName = positional[0] task.DiffFilePath = cmd.String("diff-file") task.RepairPlanPath = cmd.String("repair-plan") task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.SourceOfTruth = cmd.String("source-of-truth") task.QuietMode = cmd.Bool("quiet") task.Ctx = context.Background() task.DryRun = cmd.Bool("dry-run") task.InsertOnly = cmd.Bool("insert-only") task.UpsertOnly = cmd.Bool("upsert-only") task.FireTriggers = cmd.Bool("fire-triggers") task.FixNulls = cmd.Bool("fix-nulls") task.Bidirectional = cmd.Bool("bidirectional") task.GenerateReport = cmd.Bool("generate-report") task.RecoveryMode = cmd.Bool("recovery-mode") task.PreserveOrigin = cmd.Bool("preserve-origin") if err := task.ValidateAndPrepare(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := task.Run(true); err != nil { return fmt.Errorf("error during table repair: %w", err) } return nil } func SpockDiffCLI(cmd *cli.Command) error { args := cmd.Args().Slice() clusterName, _, err := resolveClusterArg("spock-diff", "", "[cluster]", 0, args) if err != nil { return err } task := diff.NewSpockDiffTask() task.ClusterName = clusterName task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.Output = cmd.String("output") task.Ctx = context.Background() if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.ExecuteTask(); err != nil { return fmt.Errorf("error during spock diff: %w", err) } return nil } // SchemaDiffCLI runs the schema-diff command once, or on a schedule with // --schedule. In a single run, SIGINT or SIGTERM stops it, and the error // then names the signal. With --schedule, a signal ends the schedule: a run // in progress fails with the signal in its logged error, and the command // ends without an error. func SchemaDiffCLI(cmd *cli.Command) (err error) { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("schema-diff", "", "[cluster] ", 1, args) if err != nil { return err } blockSizeStr := cmd.String("block-size") blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64) if err != nil { return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err) } task := diff.NewSchemaDiffTask() task.ClusterName = clusterName task.SchemaName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.SkipTables = cmd.String("skip-tables") scheduleEnabled := cmd.Bool("schedule") scheduleEvery := cmd.String("every") task.SkipFile = cmd.String("skip-file") task.Quiet = cmd.Bool("quiet") task.DDLOnly = cmd.Bool("ddl-only") task.Compare = cmd.String("compare") ctx, stop := signalContext(context.Background()) defer stop() defer func() { err = withStopCause(ctx, err) }() task.Ctx = ctx if scheduleEnabled && task.DDLOnly { return fmt.Errorf("scheduling is only supported when --ddl-only is false") } if scheduleEnabled && task.Compare == diff.CompareStructure { return fmt.Errorf("scheduling is not yet supported with --compare=structure") } task.BlockSize = int(blockSizeInt) task.ConcurrencyFactor = cmd.Float64("concurrency-factor") task.MaxConnections = cmd.Int("max-connections") if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil { return err } if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil { return err } task.Slicer = cmd.String("slicer") task.CompareUnitSize = cmd.Int("compare-unit-size") task.Output = cmd.String("output") task.OutputExplicit = cmd.IsSet("output") task.OverrideBlockSize = cmd.Bool("override-block-size") if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if !scheduleEnabled { if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := task.SchemaTableDiff(); err != nil { return fmt.Errorf("error during schema diff: %w", err) } return nil } freq, err := scheduler.ParseFrequency(scheduleEvery) if err != nil { return err } job := scheduler.Job{ Name: fmt.Sprintf("schema-diff:%s", task.SchemaName), Frequency: freq, RunOnStart: true, Task: func(runCtx context.Context) error { runTask := task.CloneForSchedule(runCtx) if err := runTask.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := runTask.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := runTask.SchemaTableDiff(); err != nil { return fmt.Errorf("execution failed: %w", err) } return nil }, } return scheduler.RunSingleJob(ctx, job) } // RepsetDiffCLI runs the repset-diff command once, or on a schedule with // --schedule. In a single run, SIGINT or SIGTERM stops it, and the error // then names the signal. With --schedule, a signal ends the schedule: a run // in progress fails with the signal in its logged error, and the command // ends without an error. func RepsetDiffCLI(cmd *cli.Command) (err error) { args := cmd.Args().Slice() clusterName, positional, err := resolveClusterArg("repset-diff", "", "[cluster] ", 1, args) if err != nil { return err } blockSizeStr := cmd.String("block-size") blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64) if err != nil { return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err) } scheduleEnabled := cmd.Bool("schedule") scheduleEvery := cmd.String("every") task := diff.NewRepsetDiffTask() task.ClusterName = clusterName task.RepsetName = positional[0] task.DBName = cmd.String("dbname") task.Nodes = cmd.String("nodes") task.SkipTables = cmd.String("skip-tables") task.SkipFile = cmd.String("skip-file") task.Quiet = cmd.Bool("quiet") ctx, stop := signalContext(context.Background()) defer stop() defer func() { err = withStopCause(ctx, err) }() task.Ctx = ctx task.BlockSize = int(blockSizeInt) task.ConcurrencyFactor = cmd.Float64("concurrency-factor") task.MaxConnections = cmd.Int("max-connections") if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil { return err } if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil { return err } task.Slicer = cmd.String("slicer") task.CompareUnitSize = cmd.Int("compare-unit-size") task.Output = cmd.String("output") task.OverrideBlockSize = cmd.Bool("override-block-size") if err := task.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if !scheduleEnabled { if err := task.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := diff.RepsetDiff(task); err != nil { return fmt.Errorf("error during repset diff: %w", err) } return nil } freq, err := scheduler.ParseFrequency(scheduleEvery) if err != nil { return err } job := scheduler.Job{ Name: fmt.Sprintf("repset-diff:%s", task.RepsetName), Frequency: freq, RunOnStart: true, Task: func(runCtx context.Context) error { runTask := task.CloneForSchedule(runCtx) if err := runTask.Validate(); err != nil { return fmt.Errorf("validation failed: %w", err) } if err := runTask.RunChecks(true); err != nil { return fmt.Errorf("checks failed: %w", err) } if err := diff.RepsetDiff(runTask); err != nil { return fmt.Errorf("execution failed: %w", err) } return nil }, } return scheduler.RunSingleJob(ctx, job) } func StartSchedulerCLI(_ context.Context, cmd *cli.Command) error { cfg := config.Get() if cfg == nil { return fmt.Errorf("configuration not loaded; run inside a directory with ace.yaml or set ACE_CONFIG") } component := strings.ToLower(strings.TrimSpace(cmd.String("component"))) runScheduler := false runAPI := false switch component { case "", "all": runScheduler = true runAPI = true case "scheduler": runScheduler = true case "api": runAPI = true default: return fmt.Errorf("invalid component %q (expected scheduler, api, or all)", component) } // runCtx is canceled on SIGINT or SIGTERM – triggers a full shutdown. runCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() // sighupCh receives SIGHUP signals for in-place config reload. sighupCh := make(chan os.Signal, 1) signal.Notify(sighupCh, syscall.SIGHUP) defer signal.Stop(sighupCh) // Start the API server once: it serves on-demand requests rather than // reading scheduled job config, so a reload never needs to restart it. var apiServer *server.APIServer if runAPI { if ok, apiErr := canStartAPIServer(cfg); ok { var err error apiServer, err = server.New(cfg) if err != nil { return fmt.Errorf("api server init failed: %w", err) } go func() { if err := apiServer.Run(runCtx); err != nil && !errors.Is(err, context.Canceled) { logger.Error("api server error: %v", err) stop() } }() } else if component == "api" { return fmt.Errorf("api server requested but cannot start: %w", apiErr) } else { logger.Info("api server not started: %v", apiErr) } } if !runScheduler { // API-only mode: reload config on SIGHUP, wait for shutdown. go runConfigReloadLoop(runCtx, sighupCh, apiServer) <-runCtx.Done() return nil } // schedulerReloadLoop runs the scheduler and restarts it on each valid // SIGHUP. It returns only when runCtx is canceled (SIGINT/SIGTERM). return schedulerReloadLoop(runCtx, sighupCh, apiServer) } // schedulerReloadLoop implements SIGHUP-triggered config reload. // // Design: // 1. Build jobs from the current config and start the gocron scheduler. // 2. Block waiting for either a shutdown signal (runCtx.Done) or SIGHUP. // 3. On SIGHUP: // a. Load and validate the new config (parse YAML + dry-run job build). // b. If invalid: log the error and keep the current scheduler running. // c. If valid: cancel the per-iteration scheduler context, which causes // gocron.Shutdown() to drain all in-flight jobs before returning. // Then atomically swap in the new config and loop back to step 1. // 4. On SIGINT/SIGTERM: drain in-flight jobs via schedCancel and return. func schedulerReloadLoop( runCtx context.Context, sighupCh <-chan os.Signal, apiServer *server.APIServer, ) error { for { currentCfg := config.Get() jobs, err := scheduler.BuildJobsFromConfig(currentCfg) if err != nil { return err } if len(jobs) == 0 { logger.Info("scheduler: no enabled jobs found in configuration") } else { for _, job := range jobs { logger.Info("scheduler: registering job %s", job.Name) } } // Per-iteration context: canceled to trigger graceful scheduler drain. schedCtx, schedCancel := context.WithCancel(runCtx) schedDone := make(chan error, 1) go func() { if len(jobs) == 0 { // No jobs: park until the context is canceled. <-schedCtx.Done() schedDone <- nil return } schedDone <- scheduler.RunJobs(schedCtx, jobs) }() // Inner loop: handle repeated SIGHUPs without restarting for invalid configs. reloaded := false for !reloaded { select { case <-runCtx.Done(): // SIGINT / SIGTERM: drain in-flight ops and exit. schedCancel() if err := <-schedDone; err != nil && !errors.Is(err, context.Canceled) { return err } return nil case err := <-schedDone: // The scheduler exited on its own, not via schedCancel — unexpected, // since RunJobs normally blocks until canceled. Treat a real error // as fatal; nil or Canceled means it exited cleanly. schedCancel() if err != nil && !errors.Is(err, context.Canceled) { return err } return nil case <-sighupCh: logger.Info("scheduler: received SIGHUP – reloading configuration") newCfg, loadErr := config.Reload() if loadErr != nil { logger.Error("scheduler: config reload failed (keeping current config): %v", loadErr) continue // wait for next signal } // Validate the new config by doing a dry-run job build. if _, buildErr := scheduler.BuildJobsFromConfig(newCfg); buildErr != nil { logger.Error("scheduler: new config rejected (keeping current config): %v", buildErr) continue // wait for next signal } // New config is valid. Drain in-flight jobs, then swap. logger.Info("scheduler: new configuration valid – draining in-flight operations") schedCancel() if err := <-schedDone; err != nil && !errors.Is(err, context.Canceled) { logger.Error("scheduler: error while draining for reload: %v", err) // The scheduler exited with a real error; propagate it. return err } // Atomic config swap. config.Set(newCfg) if apiServer != nil { if err := apiServer.ReloadSecurityConfig(newCfg); err != nil { logger.Warn("scheduler: security config reload failed (mTLS config unchanged): %v", err) } } logger.Info("scheduler: configuration reloaded successfully") reloaded = true // break inner loop → outer loop restarts scheduler } } // schedCancel is always called before reaching here (either inside the // SIGTERM branch, which returns, or inside the SIGHUP branch above). schedCancel() } } func StartAPIServerCLI(_ context.Context, cmd *cli.Command) error { cfg := config.Get() if cfg == nil { return fmt.Errorf("configuration not loaded; run inside a directory with ace.yaml or set ACE_CONFIG") } if ok, err := canStartAPIServer(cfg); !ok { return err } apiServer, err := server.New(cfg) if err != nil { return err } runCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() sighupCh := make(chan os.Signal, 1) signal.Notify(sighupCh, syscall.SIGHUP) defer signal.Stop(sighupCh) go runConfigReloadLoop(runCtx, sighupCh, apiServer) return apiServer.Run(runCtx) } // runConfigReloadLoop reloads config on each SIGHUP until the channel // is closed or ctx is canceled. Used by API-only and standalone-API code paths. func runConfigReloadLoop(ctx context.Context, ch <-chan os.Signal, apiServer *server.APIServer) { for { select { case <-ctx.Done(): return case <-ch: logger.Info("api: received SIGHUP – reloading configuration") newCfg, err := config.Reload() if err != nil { logger.Error("api: config reload failed (keeping current config): %v", err) continue } if ok, valErr := canStartAPIServer(newCfg); !ok { logger.Error("api: new config rejected (keeping current config): %v", valErr) continue } config.Set(newCfg) if apiServer != nil { if err := apiServer.ReloadSecurityConfig(newCfg); err != nil { logger.Warn("api: security config reload failed (mTLS config unchanged): %v", err) } } logger.Info("api: configuration reloaded successfully") } } } func canStartAPIServer(cfg *config.Config) (bool, error) { if cfg == nil { return false, fmt.Errorf("configuration not loaded") } serverCfg := cfg.Server if strings.TrimSpace(serverCfg.TLSCertFile) == "" { return false, fmt.Errorf("server.tls_cert_file is not configured") } if strings.TrimSpace(serverCfg.TLSKeyFile) == "" { return false, fmt.Errorf("server.tls_key_file is not configured") } if strings.TrimSpace(cfg.CertAuth.CACertFile) == "" { return false, fmt.Errorf("cert_auth.ca_cert_file is not configured") } return true, nil }