// ///////////////////////////////////////////////////////////////////////////
//
// # ACE - Active Consistency Engine
//
// Copyright (C) 2023 - 2026, pgEdge (https://www.pgedge.com/)
//
// This software is released under the PostgreSQL License:
// https://opensource.org/license/postgresql
//
// ///////////////////////////////////////////////////////////////////////////
package cli
import (
"context"
_ "embed"
"errors"
"fmt"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"github.com/charmbracelet/log"
"github.com/google/uuid"
"github.com/pgedge/ace/internal/api/http"
"github.com/pgedge/ace/internal/consistency/diff"
"github.com/pgedge/ace/internal/consistency/mtree"
"github.com/pgedge/ace/internal/consistency/repair"
"github.com/pgedge/ace/internal/infra/cdc"
"github.com/pgedge/ace/internal/jobs"
utils "github.com/pgedge/ace/pkg/common"
"github.com/pgedge/ace/pkg/config"
"github.com/pgedge/ace/pkg/logger"
"github.com/urfave/cli/v3"
)
//go:embed default_config.yaml
var defaultConfigYAML string
//go:embed default_pg_service.conf
var defaultPgServiceConf string
// SetupCLI builds the ace command with all its subcommands and flags.
func SetupCLI(version string) *cli.Command {
// Use -V for version, so -v stays reserved for the debug/verbose flag on
// subcommands, matching go tooling conventions.
cli.VersionFlag = &cli.BoolFlag{
Name: "version",
Aliases: []string{"V"},
Usage: "print the version",
}
commonFlags := []cli.Flag{
&cli.StringFlag{
Name: "dbname",
Aliases: []string{"d"},
Usage: "Name of the database",
Value: "",
},
&cli.StringFlag{
Name: "nodes",
Aliases: []string{"n"},
Usage: "Nodes to include in the diff (default: all)",
Value: "all",
},
&cli.BoolFlag{
Name: "quiet",
Aliases: []string{"q"},
Usage: "Whether to suppress output",
Value: false,
},
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"v"},
Usage: "Enable debug logging",
Value: false,
},
}
runTimeoutFlag := &cli.IntFlag{
Name: "run-timeout",
Usage: "Seconds for the diff of one table, 0 = no limit (default: table_diff.run_timeout, or 0)",
}
diffFlags := []cli.Flag{
&cli.StringFlag{
Name: "block-size",
Aliases: []string{"b"},
Usage: "Number of rows per block",
Value: "100000",
},
&cli.Float64Flag{
Name: "concurrency-factor",
Aliases: []string{"c"},
Usage: "CPU ratio for the comparison of the blocks that differ (0.0–4.0, e.g. 0.5 uses half of available CPUs); hashing uses --max-connections",
Value: 0.5,
},
&cli.IntFlag{
Name: "compare-unit-size",
Aliases: []string{"u"},
Usage: "Deprecated: no longer used and ignored",
Value: 0,
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Usage: "Output format",
Value: "json",
},
&cli.BoolFlag{
Name: "override-block-size",
Aliases: []string{"B"},
Usage: "Override block size",
Value: false,
},
&cli.IntFlag{
Name: "max-connections",
Aliases: []string{"M"},
Usage: "Connections to each node for one table-diff task; a set value is a strict limit for the whole task (0 = max_connections from the config file; if that is not set, 8 for cutting and hashing, and the comparison can open more)",
Value: 0,
},
&cli.IntFlag{
Name: "block-timeout",
Usage: "Seconds for one query on one block, 0 = no limit (default: table_diff.block_timeout, or 60)",
},
runTimeoutFlag,
&cli.StringFlag{
Name: "slicer",
Usage: "How to cut a table into blocks: auto, cutter or sample (default: table_diff.slicer from ace.yaml, or auto)",
},
}
rerunOnlyFlags := []cli.Flag{
&cli.StringFlag{
Name: "diff-file",
Aliases: []string{"f"},
Usage: "Path to the diff result directory, or to the JSON report next to it, to rerun from (required)",
Required: true,
},
}
skipFlags := []cli.Flag{
&cli.StringFlag{
Name: "skip-tables",
Aliases: []string{"T"},
Usage: "Comma-separated list of tables to skip",
},
&cli.StringFlag{
Name: "skip-file",
Aliases: []string{"s"},
Usage: "Path to a file with a list of tables to skip",
},
}
tableDiffFlags := append(commonFlags, diffFlags...)
tableDiffFlags = append(tableDiffFlags,
&cli.Int64Flag{
Name: "max-html-rows",
Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)",
Value: 0,
},
&cli.Int64Flag{
Name: "max-html-value-length",
Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)",
Value: 0,
},
&cli.StringFlag{
Name: "table-filter",
Aliases: []string{"F"},
Usage: "Where clause expression to use while diffing tables",
Value: "",
},
&cli.StringFlag{
Name: "against-origin",
Usage: "Restrict diff to rows whose node_origin matches this Spock node id or name",
Value: "",
},
&cli.StringFlag{
Name: "until",
Usage: "Optional commit timestamp upper bound (RFC3339) for rows to include",
Value: "",
},
// The block hash no longer uses pgcrypto. The flag is kept, hidden,
// so that scripts that still pass it do not fail.
&cli.BoolFlag{
Name: "ensure-pgcrypto",
Usage: "Deprecated: has no effect; table-diff no longer needs pgcrypto",
Value: false,
Hidden: true,
},
&cli.BoolFlag{
Name: "schedule",
Aliases: []string{"S"},
Usage: "Schedule a table-diff job to run periodically",
Value: false,
},
&cli.StringFlag{
Name: "every",
Aliases: []string{"e"},
Usage: "Time duration (e.g., 5m, 3h, etc.)",
Value: "",
},
)
diffReportFlags := []cli.Flag{
&cli.StringFlag{
Name: "diff-file",
Aliases: []string{"f"},
Usage: "Path to the diff result directory, or to the JSON report next to it (required)",
Required: true,
},
&cli.Int64Flag{
Name: "max-html-rows",
Usage: "Max rows per node pair in the HTML report (0 = use table_diff.max_html_rows from config, default 10000)",
Value: 0,
},
&cli.Int64Flag{
Name: "max-html-value-length",
Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)",
Value: 0,
},
&cli.BoolFlag{
Name: "quiet",
Aliases: []string{"q"},
Usage: "Whether to suppress output",
Value: false,
},
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"v"},
Usage: "Enable debug logging",
Value: false,
},
}
tableRerunFlags := append(commonFlags, rerunOnlyFlags...)
tableRerunFlags = append(tableRerunFlags, runTimeoutFlag)
tableRepairFlags := []cli.Flag{
&cli.StringFlag{
Name: "diff-file",
Aliases: []string{"f"},
Usage: "Path to the diff result directory, or to the JSON report next to it (required)",
Required: true,
},
&cli.StringFlag{
Name: "repair-plan",
Aliases: []string{"p"},
Usage: "Path to the advanced repair plan (YAML/JSON); skips source-of-truth requirement",
},
&cli.StringFlag{
Name: "source-of-truth",
Aliases: []string{"r"},
Usage: "Name of the node to be considered the source of truth",
},
&cli.BoolFlag{
Name: "dry-run",
Aliases: []string{"y"},
Usage: "Show what would be done without executing",
Value: false,
},
&cli.BoolFlag{
Name: "generate-report",
Aliases: []string{"g"},
Usage: "Generate a report of the repair operation",
Value: false,
},
&cli.BoolFlag{
Name: "insert-only",
Aliases: []string{"i"},
Usage: "Only perform inserts, no updates or deletes",
Value: false,
},
&cli.BoolFlag{
Name: "upsert-only",
Aliases: []string{"P"},
Usage: "Only perform upserts (insert or update), no deletes",
Value: false,
},
&cli.BoolFlag{
Name: "fire-triggers",
Aliases: []string{"t"},
Usage: "Whether to fire triggers during repairs",
Value: false,
},
&cli.BoolFlag{
Name: "bidirectional",
Aliases: []string{"Z"},
Usage: "Whether to perform repairs in both directions. Can be used only with the insert-only option",
Value: false,
},
&cli.BoolFlag{
Name: "recovery-mode",
Usage: "Enable recovery-mode repair using origin-only diffs",
Value: false,
},
&cli.BoolFlag{
Name: "preserve-origin",
Usage: "Preserve replication origin node ID and commit timestamp for repaired rows",
Value: false,
},
&cli.BoolFlag{
Name: "fix-nulls",
Aliases: []string{"X"},
Usage: "Fill NULL columns on each node using non-NULL values from its peers (no source-of-truth required)",
Value: false,
},
}
tableRepairFlags = append(commonFlags, tableRepairFlags...)
spockDiffFlags := append(commonFlags, &cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Usage: "Output format",
Value: "json",
})
repsetDiffFlags := append(commonFlags, diffFlags...)
repsetDiffFlags = append(repsetDiffFlags, skipFlags...)
repsetDiffFlags = append(repsetDiffFlags,
&cli.BoolFlag{
Name: "schedule",
Aliases: []string{"S"},
Usage: "Schedule a repset-diff job to run periodically",
Value: false,
},
&cli.StringFlag{
Name: "every",
Aliases: []string{"e"},
Usage: "Time duration (e.g., 5m, 3h, etc.)",
Value: "",
},
)
schemaDiffFlags := append(commonFlags, diffFlags...)
schemaDiffFlags = append(schemaDiffFlags, skipFlags...)
schemaDiffFlags = append(schemaDiffFlags, &cli.BoolFlag{
Name: "ddl-only",
Aliases: []string{"L"},
Usage: "Compare only schema objects (tables, functions, etc.), not table data",
Value: false,
},
&cli.StringFlag{
Name: "compare",
Usage: "What to compare: \"data\" (default, per-table data diff) or \"structure\" (symmetric column/key/constraint comparison, exits nonzero with a severity-coded status when something differs)",
Value: diff.CompareData,
},
&cli.BoolFlag{
Name: "schedule",
Aliases: []string{"S"},
Usage: "Schedule a schema-diff job to run periodically",
Value: false,
},
&cli.StringFlag{
Name: "every",
Aliases: []string{"e"},
Usage: "Time duration (e.g., 5m, 3h, etc.)",
Value: "",
},
)
mtreeBuildFlags := []cli.Flag{
&cli.StringFlag{
Name: "block-size",
Aliases: []string{"b"},
Usage: "Rows per leaf block",
Value: "10000",
},
&cli.Float64Flag{
Name: "max-cpu-ratio",
Aliases: []string{"m"},
Usage: "Share of this host's CPUs to use as parallel workers per node",
Value: 0.5,
},
&cli.IntFlag{
Name: "max-connections",
Aliases: []string{"M"},
Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)",
Value: 0,
},
&cli.BoolFlag{
Name: "override-block-size",
Aliases: []string{"B"},
Usage: "Skip block size check, and potentially tolerate unsafe block sizes",
Value: false,
},
&cli.BoolFlag{
Name: "analyse",
Aliases: []string{"a"},
Usage: "Run ANALYZE on the table",
Value: false,
},
&cli.BoolFlag{
Name: "recreate-objects",
Aliases: []string{"R"},
Usage: "Drop and recreate Merkle tree objects",
Value: false,
},
&cli.BoolFlag{
Name: "write-ranges",
Aliases: []string{"w"},
Usage: "Write block ranges to a JSON file",
Value: false,
},
&cli.StringFlag{
Name: "ranges-file",
Aliases: []string{"k"},
Usage: "Path to a file with pre-computed ranges",
},
}
mtreeBuildFlags = append(mtreeBuildFlags, commonFlags...)
mtreeUpdateFlags := []cli.Flag{
&cli.Float64Flag{
Name: "max-cpu-ratio",
Aliases: []string{"m"},
Usage: "Share of this host's CPUs to use as parallel workers per node",
Value: 0.5,
},
&cli.IntFlag{
Name: "max-connections",
Aliases: []string{"M"},
Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)",
Value: 0,
},
&cli.BoolFlag{
Name: "rebalance",
Aliases: []string{"l"},
Usage: "Rebalance the tree by merging small blocks",
Value: false,
},
&cli.BoolFlag{
Name: "skip-cdc",
Aliases: []string{"U"},
Usage: "Skip CDC processing (only rehash dirty blocks)",
Value: false,
},
&cli.IntFlag{
Name: "cdc-timeout",
Usage: "Seconds to spend draining CDC before giving up (0 = use cdc_processing_timeout / default)",
Value: 0,
},
}
mtreeUpdateFlags = append(mtreeUpdateFlags, commonFlags...)
mtreeDiffFlags := []cli.Flag{
&cli.Int64Flag{
Name: "max-html-rows",
Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)",
Value: 0,
},
&cli.Int64Flag{
Name: "max-html-value-length",
Usage: "Max characters of a value in the HTML report (0 = use max_html_value_length from config, default 1024)",
Value: 0,
},
&cli.Float64Flag{
Name: "max-cpu-ratio",
Aliases: []string{"m"},
Usage: "Share of this host's CPUs to use as parallel workers per node",
Value: 0.5,
},
&cli.IntFlag{
Name: "max-connections",
Aliases: []string{"M"},
Usage: "Maximum number of database connections per node (0 = mtree.max_connections from config, else derived from --max-cpu-ratio)",
Value: 0,
},
&cli.StringFlag{
Name: "until",
Usage: "Optional commit timestamp upper bound (RFC3339) for rows to include",
Value: "",
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Usage: "Output format",
Value: "json",
},
&cli.BoolFlag{
Name: "skip-cdc",
Aliases: []string{"U"},
Usage: "Skip CDC processing (only rehash dirty blocks and compare)",
Value: false,
},
&cli.IntFlag{
Name: "cdc-timeout",
Usage: "Seconds to spend draining CDC before giving up (0 = use cdc_processing_timeout / default)",
Value: 0,
},
}
mtreeDiffFlags = append(mtreeDiffFlags, commonFlags...)
configInitFlags := []cli.Flag{
&cli.StringFlag{
Name: "path",
Aliases: []string{"p"},
Usage: "Path to write the config file",
Value: "ace.yaml",
},
&cli.BoolFlag{
Name: "force",
Aliases: []string{"x"},
Usage: "Overwrite the config file if it already exists",
},
&cli.BoolFlag{
Name: "stdout",
Aliases: []string{"z"},
Usage: "Print the config to stdout instead of writing a file",
},
}
clusterInitFlags := []cli.Flag{
&cli.StringFlag{
Name: "path",
Aliases: []string{"p"},
Usage: "Path to write the pg service file",
Value: "pg_service.conf",
},
&cli.BoolFlag{
Name: "force",
Aliases: []string{"x"},
Usage: "Overwrite the service file if it already exists",
},
&cli.BoolFlag{
Name: "stdout",
Aliases: []string{"z"},
Usage: "Print the service file to stdout instead of writing a file",
},
}
app := &cli.Command{
Name: "ace",
Usage: "ACE - Active Consistency Engine",
Version: version,
Commands: []*cli.Command{
{
Name: "config",
Usage: "Manage ACE configuration files",
Commands: []*cli.Command{
{
Name: "init",
Usage: "Create a default ace.yaml file",
Flags: configInitFlags,
Action: ConfigInitCLI,
},
},
},
{
Name: "start",
Usage: "Start the ACE scheduler for configured jobs",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"v"},
Usage: "Enable debug logging",
},
&cli.StringFlag{
Name: "component",
Aliases: []string{"C"},
Usage: "Component to start: scheduler, api, or all",
Value: "all",
},
},
Action: StartSchedulerCLI,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "server",
Usage: "Run the ACE REST API server",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"v"},
Usage: "Enable debug logging",
},
},
Action: StartAPIServerCLI,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "cluster",
Usage: "Manage ACE cluster service definitions",
Commands: []*cli.Command{
{
Name: "init",
Usage: "Create a sample pg_service.conf file",
Flags: clusterInitFlags,
Action: ClusterInitCLI,
},
},
},
{
Name: "table-diff",
Usage: "Compare tables between PostgreSQL databases",
ArgsUsage: "[cluster]
",
Description: "A tool for comparing tables between PostgreSQL databases " +
"and detecting data inconsistencies",
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for table-diff: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for table-diff (usage: [cluster] )")
}
return TableDiffCLI(cmd)
},
Flags: tableDiffFlags,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "table-rerun",
Usage: "Re-run a diff from a file to check for persistent differences",
ArgsUsage: "[cluster]",
Flags: tableRerunFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 1 {
return fmt.Errorf("unexpected arguments for table-rerun (usage: [cluster])")
}
return TableRerunCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "diff-report",
Usage: "Build the HTML report of a finished diff from its result directory",
Description: "Reads the result directory of table-diff, mtree table-diff, repset-diff " +
"or schema-diff and writes the HTML report next to it. No database connection is needed.",
Flags: diffReportFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 0 {
return fmt.Errorf("unexpected arguments for diff-report (usage: --diff-file )")
}
return DiffReportCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
switch {
case cmd.Bool("debug"):
logger.SetLevel(log.DebugLevel)
case cmd.Bool("quiet"):
logger.SetLevel(log.WarnLevel)
default:
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "table-repair",
Usage: "Repair table inconsistencies based on a diff file",
ArgsUsage: "[cluster] ",
Flags: tableRepairFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for table-repair: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for table-repair (usage: [cluster] )")
}
return TableRepairCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "spock-diff",
Usage: "Compare spock metadata across cluster nodes",
ArgsUsage: "[cluster]",
Flags: spockDiffFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 1 {
return fmt.Errorf("unexpected arguments for spock-diff (usage: [cluster])")
}
return SpockDiffCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "schema-diff",
Usage: "Compare schemas across cluster nodes",
ArgsUsage: "[cluster] ",
Flags: schemaDiffFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for schema-diff: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for schema-diff (usage: [cluster] )")
}
return SchemaDiffCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "repset-diff",
Usage: "Compare replication sets across cluster nodes",
ArgsUsage: "[cluster] ",
Flags: repsetDiffFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for repset-diff: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for repset-diff (usage: [cluster] )")
}
return RepsetDiffCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "mtree",
Usage: "Merkle tree operations",
Commands: []*cli.Command{
{
Name: "init",
Usage: "Initialise Merkle tree replication for a cluster",
ArgsUsage: "[cluster]",
Flags: commonFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 1 {
return fmt.Errorf("unexpected arguments for mtree init (usage: [cluster])")
}
return MtreeInitCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "listen",
Usage: "Listen for changes and update Merkle trees",
ArgsUsage: "[cluster]",
Flags: commonFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 1 {
return fmt.Errorf("unexpected arguments for mtree listen (usage: [cluster])")
}
return MtreeListenCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "teardown",
Usage: "Teardown Merkle tree replication for a cluster",
ArgsUsage: "[cluster]",
Flags: commonFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 1 {
return fmt.Errorf("unexpected arguments for mtree teardown (usage: [cluster])")
}
return MtreeTeardownCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "teardown-table",
Usage: "Teardown Merkle tree objects for a specific table",
ArgsUsage: "[cluster] ",
Flags: commonFlags,
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for mtree teardown-table: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for mtree teardown-table (usage: [cluster] )")
}
return MtreeTeardownTableCLI(cmd)
},
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "build",
Usage: "Build a new Merkle tree for a table",
ArgsUsage: "[cluster] ",
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for mtree build: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for mtree build (usage: [cluster] )")
}
return MtreeBuildCLI(cmd)
},
Flags: mtreeBuildFlags,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "update",
Usage: "Update an existing Merkle tree for a table",
ArgsUsage: "[cluster] ",
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for mtree update: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for mtree update (usage: [cluster] )")
}
return MtreeUpdateCLI(cmd)
},
Flags: mtreeUpdateFlags,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
{
Name: "table-diff",
Usage: "Use Merkle Trees for performing table-diff",
ArgsUsage: "[cluster] ",
Action: func(_ context.Context, cmd *cli.Command) error {
argsLen := cmd.Args().Len()
if argsLen == 0 {
return fmt.Errorf("missing required argument for mtree diff: needs ")
}
if argsLen > 2 {
return fmt.Errorf("unexpected arguments for mtree diff (usage: [cluster] )")
}
return MtreeDiffCLI(cmd)
},
Flags: mtreeDiffFlags,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
if cmd.Bool("debug") {
logger.SetLevel(log.DebugLevel)
} else {
logger.SetLevel(log.InfoLevel)
}
return ctx, nil
},
},
},
},
},
}
return app
}
func initTemplateFile(cmd *cli.Command, content string, defaultPath string, label string, perm os.FileMode) error {
outputPath := cmd.String("path")
if outputPath == "" {
outputPath = defaultPath
}
if cmd.Bool("stdout") || outputPath == "-" {
fmt.Println(content)
return nil
}
if !cmd.Bool("force") {
if _, err := os.Stat(outputPath); err == nil {
return fmt.Errorf("%s already exists at %s (use --force to overwrite)", label, outputPath)
} else if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("unable to verify existing %s at %s: %w", label, outputPath, err)
}
}
dir := filepath.Dir(outputPath)
if dir != "." {
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err)
}
}
// Restrict the file before any content reaches it. pg_service.conf holds
// database credentials, and with --force over an existing world-readable
// copy, writing first and chmod'ing after would leave them exposed in
// between. Opening without O_TRUNC lets us fix the mode, then empty the
// file, then write. The Chmod is not redundant: O_CREATE's mode is
// umask-masked, and ignored outright for a file that already exists.
f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE, perm)
if err != nil {
return fmt.Errorf("failed to create %s at %s: %w", label, outputPath, err)
}
if err := f.Chmod(perm); err != nil {
f.Close()
return fmt.Errorf("failed to set permissions on %s: %w", outputPath, err)
}
if err := f.Truncate(0); err != nil {
f.Close()
return fmt.Errorf("failed to truncate %s: %w", outputPath, err)
}
if _, err := f.WriteString(content); err != nil {
f.Close()
return fmt.Errorf("failed to write %s to %s: %w", label, outputPath, err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("failed to write %s to %s: %w", label, outputPath, err)
}
fmt.Printf("Wrote %s to %s\n", label, outputPath)
return nil
}
func ConfigInitCLI(_ context.Context, cmd *cli.Command) error {
return initTemplateFile(cmd, defaultConfigYAML, "ace.yaml", "config file", 0o644)
}
func ClusterInitCLI(_ context.Context, cmd *cli.Command) error {
return initTemplateFile(cmd, defaultPgServiceConf, "pg_service.conf", "pg service file", 0o600)
}
func resolveClusterArg(cmd, missingUsage, argsUsage string, required int, args []string) (string, []string, error) {
if len(args) < required {
if required == 1 {
return "", nil, fmt.Errorf("missing required argument for %s: needs %s", cmd, missingUsage)
}
return "", nil, fmt.Errorf("missing required arguments for %s: needs %s", cmd, missingUsage)
}
if len(args) == required {
cluster := config.DefaultCluster()
if cluster == "" {
if required > 0 {
// The positional(s) supplied were consumed as the required entity
// (e.g. ), leaving the cluster unset. Spell that out so a
// lone argument isn't misread as the cluster.
return "", nil, fmt.Errorf(
"cluster name is required: %q was read as the %s argument (usage: %s %s); "+
"pass the cluster as the first argument or set default_cluster in ace.yaml",
strings.Join(args, " "), missingUsage, cmd, argsUsage)
}
return "", nil, fmt.Errorf("cluster name is required: specify one explicitly or set default_cluster in ace.yaml")
}
return cluster, args, nil
}
if len(args) == required+1 {
cluster := strings.TrimSpace(args[0])
if cluster == "" {
return "", nil, fmt.Errorf("cluster name is required: specify one explicitly or set default_cluster in ace.yaml")
}
return cluster, args[1:], nil
}
return "", nil, fmt.Errorf("unexpected arguments for %s (usage: %s)", cmd, argsUsage)
}
// timeoutFlag returns the value of a timeout flag in seconds, or nil if the
// flag is not given, so that the value from ace.yaml is used.
func timeoutFlag(cmd *cli.Command, name string) (*int, error) {
if !cmd.IsSet(name) {
return nil, nil
}
v := cmd.Int(name)
if v < 0 {
return nil, fmt.Errorf("--%s must be >= 0, got %d", name, v)
}
return &v, nil
}
// TableDiffCLI runs the table-diff command once, or on a schedule with
// --schedule. In a single run, SIGINT or SIGTERM stops it, and the error
// then names the signal. With --schedule, a signal ends the schedule: a run
// in progress fails with the signal in its logged error, and the command
// ends without an error.
func TableDiffCLI(cmd *cli.Command) (err error) {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("table-diff", "", "[cluster] ", 1, args)
if err != nil {
return err
}
blockSizeStr := cmd.String("block-size")
blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64)
if err != nil {
return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err)
}
task := diff.NewTableDiffTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DBName = cmd.String("dbname")
task.BlockSize = int(blockSizeInt)
task.ConcurrencyFactor = cmd.Float64("concurrency-factor")
task.MaxConnections = cmd.Int("max-connections")
if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil {
return err
}
if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil {
return err
}
task.Slicer = cmd.String("slicer")
task.CompareUnitSize = cmd.Int("compare-unit-size")
task.Output = strings.ToLower(cmd.String("output"))
task.MaxHTMLRows = cmd.Int64("max-html-rows")
task.MaxHTMLValueLength = cmd.Int64("max-html-value-length")
task.Nodes = cmd.String("nodes")
if cmd.Bool("ensure-pgcrypto") {
logger.Warn("--ensure-pgcrypto has no effect: table-diff no longer needs pgcrypto")
}
scheduleEnabled := cmd.Bool("schedule")
scheduleEvery := cmd.String("every")
task.TableFilter = cmd.String("table-filter")
task.AgainstOrigin = cmd.String("against-origin")
task.Until = cmd.String("until")
task.QuietMode = cmd.Bool("quiet")
task.OverrideBlockSize = cmd.Bool("override-block-size")
ctx, stop := signalContext(context.Background())
defer stop()
defer func() { err = withStopCause(ctx, err) }()
task.Ctx = ctx
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if !scheduleEnabled {
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.ExecuteTask(); err != nil {
return fmt.Errorf("error during comparison: %w", err)
}
return nil
}
freq, err := scheduler.ParseFrequency(scheduleEvery)
if err != nil {
return err
}
job := scheduler.Job{
Name: fmt.Sprintf("table-diff:%s", task.QualifiedTableName),
Frequency: freq,
RunOnStart: true,
Task: func(runCtx context.Context) error {
runTask := task.CloneForSchedule(runCtx)
if err := runTask.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := runTask.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := runTask.ExecuteTask(); err != nil {
return fmt.Errorf("error during comparison: %w", err)
}
return nil
},
}
return scheduler.RunSingleJob(ctx, job)
}
func MtreeInitCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, _, err := resolveClusterArg("mtree init", "", "[cluster]", 0, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.Mode = "init"
task.Ctx = context.Background()
if err := task.MtreeInit(); err != nil {
return fmt.Errorf("error during mtree init: %w", err)
}
return nil
}
func MtreeListenCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, _, err := resolveClusterArg("mtree listen", "", "[cluster]", 0, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.Mode = "listen"
task.Ctx = context.Background()
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
appCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// Handle graceful shutdown
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
go func() {
<-sigChan
logger.Info("Shutdown signal received, stopping listeners...")
cancel()
}()
var wg sync.WaitGroup
clusterNodes := task.GetClusterNodes()
wg.Add(len(clusterNodes))
for _, nodeInfo := range clusterNodes {
go func(ni map[string]any) {
defer wg.Done()
cdc.ListenForChanges(appCtx, ni)
}(nodeInfo)
}
logger.Info("CDC listeners started. Press Ctrl+C to stop.")
wg.Wait()
logger.Info("All CDC listeners stopped.")
return nil
}
func MtreeTeardownCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, _, err := resolveClusterArg("mtree teardown", "", "[cluster]", 0, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.Mode = "teardown"
task.Ctx = context.Background()
if err := task.MtreeTeardown(); err != nil {
return fmt.Errorf("error during mtree teardown: %w", err)
}
return nil
}
func MtreeTeardownTableCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("mtree teardown-table", "", "[cluster] ", 1, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.Mode = "teardown-table"
task.Ctx = context.Background()
if err := task.MtreeTeardownTable(); err != nil {
return fmt.Errorf("error during mtree table teardown: %w", err)
}
return nil
}
func MtreeBuildCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("mtree build", "", "[cluster] ", 1, args)
if err != nil {
return err
}
blockSizeStr := cmd.String("block-size")
blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64)
if err != nil {
return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err)
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.BlockSize = int(blockSizeInt)
task.MaxCpuRatio = cmd.Float64("max-cpu-ratio")
task.MaxConnections = cmd.Int("max-connections")
task.OverrideBlockSize = cmd.Bool("override-block-size")
task.Analyse = cmd.Bool("analyse")
task.RecreateObjects = cmd.Bool("recreate-objects")
task.WriteRanges = cmd.Bool("write-ranges")
task.RangesFile = cmd.String("ranges-file")
task.Mode = "build"
task.Ctx = context.Background()
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.BuildMtree(); err != nil {
return fmt.Errorf("error during merkle tree build: %w", err)
}
return nil
}
func MtreeUpdateCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("mtree update", "", "[cluster] ", 1, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.MaxCpuRatio = cmd.Float64("max-cpu-ratio")
task.MaxConnections = cmd.Int("max-connections")
task.Rebalance = cmd.Bool("rebalance")
task.NoCDC = cmd.Bool("skip-cdc")
task.CDCTimeoutSec = cmd.Int("cdc-timeout")
task.Mode = "update"
task.Ctx = context.Background()
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := task.RunChecks(true /* skipValidation */); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.UpdateMtree(true /* skipAllChecks */); err != nil {
return fmt.Errorf("error during merkle tree update: %w", err)
}
return nil
}
func MtreeDiffCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("mtree diff", "", "[cluster] ", 1, args)
if err != nil {
return err
}
task := mtree.NewMerkleTreeTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
task.MaxCpuRatio = cmd.Float64("max-cpu-ratio")
task.MaxConnections = cmd.Int("max-connections")
task.Output = cmd.String("output")
task.MaxHTMLRows = cmd.Int64("max-html-rows")
task.MaxHTMLValueLength = cmd.Int64("max-html-value-length")
task.NoCDC = cmd.Bool("skip-cdc")
task.CDCTimeoutSec = cmd.Int("cdc-timeout")
task.Until = cmd.String("until")
task.Mode = "diff"
task.Ctx = context.Background()
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.DiffMtree(); err != nil {
return fmt.Errorf("error during merkle tree diff: %w", err)
}
return nil
}
// TableRerunCLI compares again the rows listed in a diff file. SIGINT or
// SIGTERM stops it, and the error then names the signal.
func TableRerunCLI(cmd *cli.Command) (err error) {
args := cmd.Args().Slice()
clusterName, _, err := resolveClusterArg("table-rerun", "", "[cluster]", 0, args)
if err != nil {
return err
}
task := diff.NewTableDiffTask()
task.TaskID = uuid.NewString()
task.Mode = "rerun"
task.ClusterName = clusterName
task.DiffFilePath = cmd.String("diff-file")
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.QuietMode = cmd.Bool("quiet")
if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil {
return err
}
ctx, stop := signalContext(context.Background())
defer stop()
defer func() { err = withStopCause(ctx, err) }()
task.Ctx = ctx
if err := task.ExecuteTask(); err != nil {
return fmt.Errorf("error during table-rerun: %w", err)
}
return nil
}
// DiffReportCLI builds the HTML report of a finished diff from its result
// directory. A limit of 0 is taken from the table_diff section of the config
// file, and then from the defaults.
func DiffReportCLI(cmd *cli.Command) error {
maxRows := cmd.Int64("max-html-rows")
maxLen := cmd.Int64("max-html-value-length")
if maxRows < 0 {
return fmt.Errorf("--max-html-rows must be >= 0")
}
if maxLen < 0 {
return fmt.Errorf("--max-html-value-length must be >= 0")
}
if cfg := config.Get(); cfg != nil {
if cfg.TableDiff.MaxHTMLRows < 0 {
return fmt.Errorf("table_diff.max_html_rows in the config must be >= 0, got %d", cfg.TableDiff.MaxHTMLRows)
}
if cfg.TableDiff.MaxHTMLValueLength < 0 {
return fmt.Errorf("table_diff.max_html_value_length in the config must be >= 0, got %d", cfg.TableDiff.MaxHTMLValueLength)
}
if maxRows == 0 && cfg.TableDiff.MaxHTMLRows > 0 {
maxRows = cfg.TableDiff.MaxHTMLRows
}
if maxLen == 0 && cfg.TableDiff.MaxHTMLValueLength > 0 {
maxLen = cfg.TableDiff.MaxHTMLValueLength
}
}
r, err := utils.OpenDiffResult(cmd.String("diff-file"))
if err != nil {
return err
}
ctx, stop := signalContext(context.Background())
defer stop()
path, err := utils.WriteHTMLDiffReport(ctx, r, maxRows, maxLen)
if err != nil {
return withStopCause(ctx, err)
}
logger.Info("HTML diff report written to %s", path)
return nil
}
func TableRepairCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("table-repair", "", "[cluster] ", 1, args)
if err != nil {
return err
}
task := repair.NewTableRepairTask()
task.ClusterName = clusterName
task.QualifiedTableName = positional[0]
task.DiffFilePath = cmd.String("diff-file")
task.RepairPlanPath = cmd.String("repair-plan")
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.SourceOfTruth = cmd.String("source-of-truth")
task.QuietMode = cmd.Bool("quiet")
task.Ctx = context.Background()
task.DryRun = cmd.Bool("dry-run")
task.InsertOnly = cmd.Bool("insert-only")
task.UpsertOnly = cmd.Bool("upsert-only")
task.FireTriggers = cmd.Bool("fire-triggers")
task.FixNulls = cmd.Bool("fix-nulls")
task.Bidirectional = cmd.Bool("bidirectional")
task.GenerateReport = cmd.Bool("generate-report")
task.RecoveryMode = cmd.Bool("recovery-mode")
task.PreserveOrigin = cmd.Bool("preserve-origin")
if err := task.ValidateAndPrepare(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := task.Run(true); err != nil {
return fmt.Errorf("error during table repair: %w", err)
}
return nil
}
func SpockDiffCLI(cmd *cli.Command) error {
args := cmd.Args().Slice()
clusterName, _, err := resolveClusterArg("spock-diff", "", "[cluster]", 0, args)
if err != nil {
return err
}
task := diff.NewSpockDiffTask()
task.ClusterName = clusterName
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.Output = cmd.String("output")
task.Ctx = context.Background()
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.ExecuteTask(); err != nil {
return fmt.Errorf("error during spock diff: %w", err)
}
return nil
}
// SchemaDiffCLI runs the schema-diff command once, or on a schedule with
// --schedule. In a single run, SIGINT or SIGTERM stops it, and the error
// then names the signal. With --schedule, a signal ends the schedule: a run
// in progress fails with the signal in its logged error, and the command
// ends without an error.
func SchemaDiffCLI(cmd *cli.Command) (err error) {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("schema-diff", "", "[cluster] ", 1, args)
if err != nil {
return err
}
blockSizeStr := cmd.String("block-size")
blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64)
if err != nil {
return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err)
}
task := diff.NewSchemaDiffTask()
task.ClusterName = clusterName
task.SchemaName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.SkipTables = cmd.String("skip-tables")
scheduleEnabled := cmd.Bool("schedule")
scheduleEvery := cmd.String("every")
task.SkipFile = cmd.String("skip-file")
task.Quiet = cmd.Bool("quiet")
task.DDLOnly = cmd.Bool("ddl-only")
task.Compare = cmd.String("compare")
ctx, stop := signalContext(context.Background())
defer stop()
defer func() { err = withStopCause(ctx, err) }()
task.Ctx = ctx
if scheduleEnabled && task.DDLOnly {
return fmt.Errorf("scheduling is only supported when --ddl-only is false")
}
if scheduleEnabled && task.Compare == diff.CompareStructure {
return fmt.Errorf("scheduling is not yet supported with --compare=structure")
}
task.BlockSize = int(blockSizeInt)
task.ConcurrencyFactor = cmd.Float64("concurrency-factor")
task.MaxConnections = cmd.Int("max-connections")
if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil {
return err
}
if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil {
return err
}
task.Slicer = cmd.String("slicer")
task.CompareUnitSize = cmd.Int("compare-unit-size")
task.Output = cmd.String("output")
task.OutputExplicit = cmd.IsSet("output")
task.OverrideBlockSize = cmd.Bool("override-block-size")
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if !scheduleEnabled {
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := task.SchemaTableDiff(); err != nil {
return fmt.Errorf("error during schema diff: %w", err)
}
return nil
}
freq, err := scheduler.ParseFrequency(scheduleEvery)
if err != nil {
return err
}
job := scheduler.Job{
Name: fmt.Sprintf("schema-diff:%s", task.SchemaName),
Frequency: freq,
RunOnStart: true,
Task: func(runCtx context.Context) error {
runTask := task.CloneForSchedule(runCtx)
if err := runTask.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := runTask.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := runTask.SchemaTableDiff(); err != nil {
return fmt.Errorf("execution failed: %w", err)
}
return nil
},
}
return scheduler.RunSingleJob(ctx, job)
}
// RepsetDiffCLI runs the repset-diff command once, or on a schedule with
// --schedule. In a single run, SIGINT or SIGTERM stops it, and the error
// then names the signal. With --schedule, a signal ends the schedule: a run
// in progress fails with the signal in its logged error, and the command
// ends without an error.
func RepsetDiffCLI(cmd *cli.Command) (err error) {
args := cmd.Args().Slice()
clusterName, positional, err := resolveClusterArg("repset-diff", "", "[cluster] ", 1, args)
if err != nil {
return err
}
blockSizeStr := cmd.String("block-size")
blockSizeInt, err := strconv.ParseInt(blockSizeStr, 10, 64)
if err != nil {
return fmt.Errorf("invalid block size '%s': %w", blockSizeStr, err)
}
scheduleEnabled := cmd.Bool("schedule")
scheduleEvery := cmd.String("every")
task := diff.NewRepsetDiffTask()
task.ClusterName = clusterName
task.RepsetName = positional[0]
task.DBName = cmd.String("dbname")
task.Nodes = cmd.String("nodes")
task.SkipTables = cmd.String("skip-tables")
task.SkipFile = cmd.String("skip-file")
task.Quiet = cmd.Bool("quiet")
ctx, stop := signalContext(context.Background())
defer stop()
defer func() { err = withStopCause(ctx, err) }()
task.Ctx = ctx
task.BlockSize = int(blockSizeInt)
task.ConcurrencyFactor = cmd.Float64("concurrency-factor")
task.MaxConnections = cmd.Int("max-connections")
if task.BlockTimeout, err = timeoutFlag(cmd, "block-timeout"); err != nil {
return err
}
if task.RunTimeout, err = timeoutFlag(cmd, "run-timeout"); err != nil {
return err
}
task.Slicer = cmd.String("slicer")
task.CompareUnitSize = cmd.Int("compare-unit-size")
task.Output = cmd.String("output")
task.OverrideBlockSize = cmd.Bool("override-block-size")
if err := task.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if !scheduleEnabled {
if err := task.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := diff.RepsetDiff(task); err != nil {
return fmt.Errorf("error during repset diff: %w", err)
}
return nil
}
freq, err := scheduler.ParseFrequency(scheduleEvery)
if err != nil {
return err
}
job := scheduler.Job{
Name: fmt.Sprintf("repset-diff:%s", task.RepsetName),
Frequency: freq,
RunOnStart: true,
Task: func(runCtx context.Context) error {
runTask := task.CloneForSchedule(runCtx)
if err := runTask.Validate(); err != nil {
return fmt.Errorf("validation failed: %w", err)
}
if err := runTask.RunChecks(true); err != nil {
return fmt.Errorf("checks failed: %w", err)
}
if err := diff.RepsetDiff(runTask); err != nil {
return fmt.Errorf("execution failed: %w", err)
}
return nil
},
}
return scheduler.RunSingleJob(ctx, job)
}
func StartSchedulerCLI(_ context.Context, cmd *cli.Command) error {
cfg := config.Get()
if cfg == nil {
return fmt.Errorf("configuration not loaded; run inside a directory with ace.yaml or set ACE_CONFIG")
}
component := strings.ToLower(strings.TrimSpace(cmd.String("component")))
runScheduler := false
runAPI := false
switch component {
case "", "all":
runScheduler = true
runAPI = true
case "scheduler":
runScheduler = true
case "api":
runAPI = true
default:
return fmt.Errorf("invalid component %q (expected scheduler, api, or all)", component)
}
// runCtx is canceled on SIGINT or SIGTERM – triggers a full shutdown.
runCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
// sighupCh receives SIGHUP signals for in-place config reload.
sighupCh := make(chan os.Signal, 1)
signal.Notify(sighupCh, syscall.SIGHUP)
defer signal.Stop(sighupCh)
// Start the API server once: it serves on-demand requests rather than
// reading scheduled job config, so a reload never needs to restart it.
var apiServer *server.APIServer
if runAPI {
if ok, apiErr := canStartAPIServer(cfg); ok {
var err error
apiServer, err = server.New(cfg)
if err != nil {
return fmt.Errorf("api server init failed: %w", err)
}
go func() {
if err := apiServer.Run(runCtx); err != nil && !errors.Is(err, context.Canceled) {
logger.Error("api server error: %v", err)
stop()
}
}()
} else if component == "api" {
return fmt.Errorf("api server requested but cannot start: %w", apiErr)
} else {
logger.Info("api server not started: %v", apiErr)
}
}
if !runScheduler {
// API-only mode: reload config on SIGHUP, wait for shutdown.
go runConfigReloadLoop(runCtx, sighupCh, apiServer)
<-runCtx.Done()
return nil
}
// schedulerReloadLoop runs the scheduler and restarts it on each valid
// SIGHUP. It returns only when runCtx is canceled (SIGINT/SIGTERM).
return schedulerReloadLoop(runCtx, sighupCh, apiServer)
}
// schedulerReloadLoop implements SIGHUP-triggered config reload.
//
// Design:
// 1. Build jobs from the current config and start the gocron scheduler.
// 2. Block waiting for either a shutdown signal (runCtx.Done) or SIGHUP.
// 3. On SIGHUP:
// a. Load and validate the new config (parse YAML + dry-run job build).
// b. If invalid: log the error and keep the current scheduler running.
// c. If valid: cancel the per-iteration scheduler context, which causes
// gocron.Shutdown() to drain all in-flight jobs before returning.
// Then atomically swap in the new config and loop back to step 1.
// 4. On SIGINT/SIGTERM: drain in-flight jobs via schedCancel and return.
func schedulerReloadLoop(
runCtx context.Context,
sighupCh <-chan os.Signal,
apiServer *server.APIServer,
) error {
for {
currentCfg := config.Get()
jobs, err := scheduler.BuildJobsFromConfig(currentCfg)
if err != nil {
return err
}
if len(jobs) == 0 {
logger.Info("scheduler: no enabled jobs found in configuration")
} else {
for _, job := range jobs {
logger.Info("scheduler: registering job %s", job.Name)
}
}
// Per-iteration context: canceled to trigger graceful scheduler drain.
schedCtx, schedCancel := context.WithCancel(runCtx)
schedDone := make(chan error, 1)
go func() {
if len(jobs) == 0 {
// No jobs: park until the context is canceled.
<-schedCtx.Done()
schedDone <- nil
return
}
schedDone <- scheduler.RunJobs(schedCtx, jobs)
}()
// Inner loop: handle repeated SIGHUPs without restarting for invalid configs.
reloaded := false
for !reloaded {
select {
case <-runCtx.Done():
// SIGINT / SIGTERM: drain in-flight ops and exit.
schedCancel()
if err := <-schedDone; err != nil && !errors.Is(err, context.Canceled) {
return err
}
return nil
case err := <-schedDone:
// The scheduler exited on its own, not via schedCancel — unexpected,
// since RunJobs normally blocks until canceled. Treat a real error
// as fatal; nil or Canceled means it exited cleanly.
schedCancel()
if err != nil && !errors.Is(err, context.Canceled) {
return err
}
return nil
case <-sighupCh:
logger.Info("scheduler: received SIGHUP – reloading configuration")
newCfg, loadErr := config.Reload()
if loadErr != nil {
logger.Error("scheduler: config reload failed (keeping current config): %v", loadErr)
continue // wait for next signal
}
// Validate the new config by doing a dry-run job build.
if _, buildErr := scheduler.BuildJobsFromConfig(newCfg); buildErr != nil {
logger.Error("scheduler: new config rejected (keeping current config): %v", buildErr)
continue // wait for next signal
}
// New config is valid. Drain in-flight jobs, then swap.
logger.Info("scheduler: new configuration valid – draining in-flight operations")
schedCancel()
if err := <-schedDone; err != nil && !errors.Is(err, context.Canceled) {
logger.Error("scheduler: error while draining for reload: %v", err)
// The scheduler exited with a real error; propagate it.
return err
}
// Atomic config swap.
config.Set(newCfg)
if apiServer != nil {
if err := apiServer.ReloadSecurityConfig(newCfg); err != nil {
logger.Warn("scheduler: security config reload failed (mTLS config unchanged): %v", err)
}
}
logger.Info("scheduler: configuration reloaded successfully")
reloaded = true // break inner loop → outer loop restarts scheduler
}
}
// schedCancel is always called before reaching here (either inside the
// SIGTERM branch, which returns, or inside the SIGHUP branch above).
schedCancel()
}
}
func StartAPIServerCLI(_ context.Context, cmd *cli.Command) error {
cfg := config.Get()
if cfg == nil {
return fmt.Errorf("configuration not loaded; run inside a directory with ace.yaml or set ACE_CONFIG")
}
if ok, err := canStartAPIServer(cfg); !ok {
return err
}
apiServer, err := server.New(cfg)
if err != nil {
return err
}
runCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
sighupCh := make(chan os.Signal, 1)
signal.Notify(sighupCh, syscall.SIGHUP)
defer signal.Stop(sighupCh)
go runConfigReloadLoop(runCtx, sighupCh, apiServer)
return apiServer.Run(runCtx)
}
// runConfigReloadLoop reloads config on each SIGHUP until the channel
// is closed or ctx is canceled. Used by API-only and standalone-API code paths.
func runConfigReloadLoop(ctx context.Context, ch <-chan os.Signal, apiServer *server.APIServer) {
for {
select {
case <-ctx.Done():
return
case <-ch:
logger.Info("api: received SIGHUP – reloading configuration")
newCfg, err := config.Reload()
if err != nil {
logger.Error("api: config reload failed (keeping current config): %v", err)
continue
}
if ok, valErr := canStartAPIServer(newCfg); !ok {
logger.Error("api: new config rejected (keeping current config): %v", valErr)
continue
}
config.Set(newCfg)
if apiServer != nil {
if err := apiServer.ReloadSecurityConfig(newCfg); err != nil {
logger.Warn("api: security config reload failed (mTLS config unchanged): %v", err)
}
}
logger.Info("api: configuration reloaded successfully")
}
}
}
func canStartAPIServer(cfg *config.Config) (bool, error) {
if cfg == nil {
return false, fmt.Errorf("configuration not loaded")
}
serverCfg := cfg.Server
if strings.TrimSpace(serverCfg.TLSCertFile) == "" {
return false, fmt.Errorf("server.tls_cert_file is not configured")
}
if strings.TrimSpace(serverCfg.TLSKeyFile) == "" {
return false, fmt.Errorf("server.tls_key_file is not configured")
}
if strings.TrimSpace(cfg.CertAuth.CACertFile) == "" {
return false, fmt.Errorf("cert_auth.ca_cert_file is not configured")
}
return true, nil
}