#!/bin/sh
# hm -- the documented steps, in the order that works.
#
# Not a framework and not a new way to run anything: every subcommand below shells the same
# scripts docs/INSTALL.md describes. It exists because the ORDER is load-bearing and getting it
# wrong is silent. Build the ANN index before the rows exist and it is far slower; skip it and
# search still answers, by sequential-scanning every vector for the life of the install. Re-run a
# full ingest over an existing scope and it deletes the documents first, taking every embedding
# with them. Neither raises an error.
#
#   ./hm init                  bring up the docker stack, apply the migrations, print exports
#   ./hm migrate [--owner <author>] [--reapply <file>]... [--list | --status]
#                              apply sql/ in the one order that works, recording each file's
#                              SHA-256 in hm_schema_migrations. An applied file with the same
#                              checksum is skipped, so a second run is a no-op. A file that
#                              CHANGED after it was applied stops the run before anything is
#                              applied; --reapply <file> says you read the change and want it
#                              re-run. --owner claims unreferenced legacy entities (default:
#                              MEM_AUTHOR, else your login). --list prints the order, --status
#                              what is applied, pending or changed. Connects as HM_ADMIN_URL,
#                              else DATABASE_URL: it must be the role that owns the tables.
#   ./hm author add <author> [<role>]
#                              create a login the DATABASE maps to <author> (enforced identity:
#                              it cannot read or write as anyone else, whatever it sets
#                              mem.reader to) and print how to set its password
#   ./hm author remove <role>  unmap a login, revoke hm_app from it and disable it
#   ./hm author list           which logins are mapped to which author (owner only)
#   ./hm ingest <dir> <scope> [--walk]
#                              ingest -> embed -> index, incrementally if the scope already exists.
#                              --walk enumerates the tree instead of asking git, for a directory
#                              that is git-ignored or not a repository at all -- without it such a
#                              directory ingests ZERO documents and says why.
#                              A `.hmignore` in <dir> keeps paths OUT of the corpus (one
#                              pattern per line, trailing '/' = whole subtree). For the file of
#                              credentials and the superseded spec set that live next to the real
#                              documents: --walk cannot exclude them, it is all-or-nothing.
#   ./hm refresh               re-ingest every scope in $HM_SCOPES (default
#                              ~/.config/hypermnesia/scopes: <dir><TAB><scope><TAB>[--walk] per
#                              line), then assert no chunk was left without an embedding. This is
#                              the one to put on a timer -- freshness.py reports drift, this ends
#                              it.
#   ./hm install [--wire] [--ref <commit>] [--console[=<features>]] [--dry-run] ...
#                              install a COMMIT of this checkout as the machine's release:
#                              ~/.HyperMnesia/current (HYPERMNESIA_HOME moves it), with the MCP
#                              server built into it. Hooks, jobs and MCP configs then run that,
#                              not this working copy. --wire rewrites the configs that still point
#                              into a checkout (each one backed up first). ci/install.py --help.
#   ./hm doctor                is this install actually working, or only appearing to
#   ./hm latency               where the time goes: hook, embedder, database, reranker
#
# Env: DATABASE_URL (required by ingest/doctor; init prints the one it just set up).
#      HM_ADMIN_URL (migrate, author: the table owner's connection; default DATABASE_URL).
set -eu

ROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PY=${HM_PYTHON:-python3}

die()  { printf '%s\n' "hm: $*" >&2; exit 1; }
step() { printf '\n== %s\n' "$*"; }
need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not on PATH"; }

# The header comment above IS the help text; printing it from the file keeps the two from
# drifting. Stop at the first line that is not a comment rather than a hardcoded line number,
# which goes stale the moment anyone edits the header.
usage() {
    awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
    exit "${1:-1}"
}

# -- migrations -------------------------------------------------------------------------------
# THE apply order, and the only copy of it. CI, the docs and ci/doctor.py read it from here
# (`./hm migrate --list`) instead of keeping their own: there used to be four copies, and nothing
# noticed when one fell behind.
#
#   schema.sql                      documents, chunks, the component map
#   migrate_schema_version.sql      hm_schema_migrations: what has been applied, with checksums
#   schema_mem.sql .. dreamer       personal memory, then its migrations in dependency order
#   migrate_author_roles.sql        per-author logins (enforced identity)
#   mem_app_role.sql                LAST: grants on, and re-states the policies for, all of it
MIGRATIONS="schema.sql
migrate_schema_version.sql
schema_mem.sql
mem_multiuser.sql
migrate_entity_ownership.sql
migrate_memory_perspectives.sql
migrate_memory_evidence.sql
migrate_peer_cards.sql
migrate_dreamer.sql
migrate_memory_lifecycle.sql
migrate_constraint_sources.sql
migrate_author_roles.sql
mem_app_role.sql"

# Re-applied whenever anything before it was applied in the same run, and whenever it changed:
# it grants on everything above it, so a table added without re-running it is unreachable to the
# application -- which looks like an empty store, not like an error.
REPEATABLE="mem_app_role.sql"

# Tolerated when absent, with a note. Every other missing file is an error.
MAY_BE_ABSENT=""

SQL_DIR=${HM_SQL_DIR:-$ROOT/sql}

# How migrate reaches the database: `psql "$MIGRATE_URL"` from this host by default; `hm init`
# switches it to the psql inside the compose container, which needs nothing installed here.
MIGRATE_VIA=url
MIGRATE_URL=
mpsql() {
    if [ "$MIGRATE_VIA" = compose ]; then
        docker compose -f "$ROOT/deploy/docker/docker-compose.yml" exec -T postgres \
            psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"
    else
        psql "$MIGRATE_URL" "$@"
    fi
}

in_list() {   # in_list <word> <space-separated list>
    case " $2 " in *" $1 "*) return 0 ;; esac
    return 1
}

sha256_of() {
    if command -v sha256sum >/dev/null 2>&1; then s=$(sha256sum < "$1")
    elif command -v shasum >/dev/null 2>&1; then s=$(shasum -a 256 < "$1")
    else s=$(openssl dgst -sha256 -r < "$1"); fi
    s=${s%% *}
    case "$s" in ''|*[!0-9a-f]*) die "could not checksum $1" ;; esac
    [ "${#s}" -eq 64 ] || die "could not checksum $1"
    printf %s "$s"
}

recorded_sum() {   # the checksum recorded for <file>, or nothing
    awk -v n="$1" '$1 == n { print $2 }' "$mtmp/recorded"
}

record_applied() {   # record_applied <file> <sha256>
    # Both values are safe to interpolate: the name comes from MIGRATIONS, the sum is 64 hex.
    mpsql -tAX -q -v ON_ERROR_STOP=1 -c "INSERT INTO public.hm_schema_migrations (name, checksum)
        VALUES ('$1', '$2') ON CONFLICT (name)
        DO UPDATE SET checksum = EXCLUDED.checksum, applied_at = now()" >/dev/null \
        || die "sql/$1 was applied but could not be recorded -- re-run ./hm migrate"
}

cmd_migrate() {
    owner=${MEM_AUTHOR:-$(id -un)}
    reapply=""
    mode=apply
    while [ $# -gt 0 ]; do
        case "$1" in
            --list)      mode=list ;;
            --status)    mode=status ;;
            --owner)     [ $# -ge 2 ] || usage; owner=$2; shift ;;
            --owner=*)   owner=${1#--owner=} ;;
            --reapply)   [ $# -ge 2 ] || usage; reapply="$reapply ${2##*/}"; shift ;;
            --reapply=*) v=${1#--reapply=}; reapply="$reapply ${v##*/}" ;;
            *) usage ;;
        esac
        shift
    done
    if [ "$mode" = list ]; then
        printf '%s\n' $MIGRATIONS
        return 0
    fi
    all=$(printf '%s ' $MIGRATIONS)
    for r in $reapply; do
        in_list "$r" "$all" || die "--reapply $r: not a migration (see ./hm migrate --list)"
    done
    # The owner value lands in a psql variable that one migration interpolates as a literal.
    case "$owner" in
        *[!A-Za-z0-9._-]*|"") die "--owner must match [A-Za-z0-9._-]+ (got: $owner)" ;;
    esac
    if [ "$MIGRATE_VIA" = url ]; then
        need psql
        MIGRATE_URL=${HM_ADMIN_URL:-${DATABASE_URL:-}}
        [ -n "$MIGRATE_URL" ] || die "set HM_ADMIN_URL (or DATABASE_URL) to the table owner's connection string"
    fi

    mtmp=$(mktemp -d); trap 'rm -rf "$mtmp"' EXIT
    if ! has=$(mpsql -tAX -v ON_ERROR_STOP=1 \
                -c "SELECT to_regclass('public.hm_schema_migrations') IS NOT NULL" 2>&1); then
        die "could not reach the database: $has"
    fi
    : > "$mtmp/recorded"
    if [ "$has" = t ]; then
        mpsql -tAX -v ON_ERROR_STOP=1 -F ' ' \
            -c "SELECT name, checksum FROM public.hm_schema_migrations ORDER BY name" \
            > "$mtmp/recorded" || die "could not read hm_schema_migrations"
    fi

    # Pass 1: decide everything before applying anything. A changed file found halfway through
    # would leave this run half applied and half refused.
    changed=""
    for f in $MIGRATIONS; do
        path="$SQL_DIR/$f"
        if [ ! -f "$path" ]; then
            in_list "$f" "$MAY_BE_ABSENT" || die "sql/$f is missing -- this checkout is incomplete"
            [ "$mode" != status ] || printf '%-32s absent from this checkout\n' "$f"
            continue
        fi
        sum=$(sha256_of "$path")
        rec=$(recorded_sum "$f")
        state=pending
        if [ -n "$rec" ] && [ "$rec" = "$sum" ]; then
            state=applied
        elif [ -n "$rec" ]; then
            if in_list "$f" "$REPEATABLE"; then
                state="changed (re-applied by the next run)"
            elif in_list "$f" "$reapply"; then
                state="changed (--reapply)"
            else
                state=CHANGED
                changed="$changed $f"
            fi
        fi
        [ "$mode" != status ] || printf '%-32s %s\n' "$f" "$state"
    done
    [ "$mode" != status ] || return 0
    if [ -n "$changed" ]; then
        printf '!! changed since they were applied:%s\n' "$changed" >&2
        printf '!! An applied migration is not replayed silently. Read the change, then run\n' >&2
        printf '!!     ./hm migrate --reapply <file>\n' >&2
        printf '!! for each one (every file in sql/ is written to be re-runnable).\n' >&2
        die "refusing to migrate: nothing was applied"
    fi

    # Pass 2: apply. hm_schema_migrations does not exist until migrate_schema_version.sql has
    # run, so the files applied before it are recorded as soon as it has.
    recordable=$has
    owed=""
    unrecorded=""
    applied=0
    current=0
    for f in $MIGRATIONS; do
        path="$SQL_DIR/$f"
        if [ ! -f "$path" ]; then
            printf -- '-- sql/%s is not in this checkout -- skipped\n' "$f"
            continue
        fi
        sum=$(sha256_of "$path")
        rec=$(recorded_sum "$f")
        # Current, unless it was named in --reapply, or it is the grant file and something
        # before it has just been applied.
        if [ "$rec" = "$sum" ] && ! in_list "$f" "$reapply" \
           && { ! in_list "$f" "$REPEATABLE" || [ "$applied" -eq 0 ]; }; then
            current=$((current + 1))
            continue
        fi
        # Anything applied before the grant file can (re)install a policy or a table it has not
        # restated yet, and until it runs again the store is not in the shape it certifies.
        # Forget its record FIRST: a run interrupted between this file and the grant file then
        # still owes it, instead of finding every checksum current and applying nothing.
        if [ "$recordable" = t ] && ! in_list "$f" "$REPEATABLE" && [ -z "$owed" ]; then
            for r in $REPEATABLE; do
                mpsql -tAX -q -v ON_ERROR_STOP=1 \
                    -c "DELETE FROM public.hm_schema_migrations WHERE name = '$r'" >/dev/null \
                    || die "could not mark sql/$r as owed -- nothing was applied"
            done
            owed=t
        fi
        printf -- '-- applying sql/%s\n' "$f"
        # From stdin, so the same call works against the compose container, which sees the
        # files under another path. The output is shown only on failure: an idempotent re-run
        # prints a page of "already exists, skipping" notices that hide the line that matters.
        if ! mpsql -X -q -v ON_ERROR_STOP=1 -v owner="$owner" < "$path" > "$mtmp/out" 2>&1; then
            cat "$mtmp/out" >&2
            die "sql/$f failed -- nothing after it was applied"
        fi
        applied=$((applied + 1))
        [ "$f" != migrate_schema_version.sql ] || recordable=t
        unrecorded="$unrecorded $f:$sum"
        if [ "$recordable" = t ]; then
            for u in $unrecorded; do
                record_applied "${u%%:*}" "${u#*:}"
            done
            unrecorded=""
        fi
    done
    [ -z "$unrecorded" ] || die "applied but not recorded (no hm_schema_migrations):$unrecorded"
    if [ "$applied" -eq 0 ]; then
        printf -- '-- nothing to apply: all %s migration(s) are recorded with their current checksum\n' "$current"
    else
        printf '\nmigrations: %s applied, %s already current\n' "$applied" "$current"
    fi
}

# -- per-author logins ------------------------------------------------------------------------
cmd_author() {
    [ $# -ge 1 ] || usage
    sub=$1; shift
    # Arguments first, the database second: a bad name is refused before anything connects.
    case "$sub" in
        add)
            [ $# -ge 1 ] && [ $# -le 2 ] || usage
            case "$1" in *[!A-Za-z0-9._-]*|"") die "author must match [A-Za-z0-9._-]+ (got: $1)" ;; esac
            case "${2:-}" in *[!a-z0-9_]*) die "role must match [a-z0-9_]+ (got: $2)" ;; esac ;;
        remove)
            [ $# -eq 1 ] || usage
            case "$1" in *[!a-z0-9_]*|"") die "role must match [a-z0-9_]+ (got: $1)" ;; esac ;;
        list) ;;
        *) usage ;;
    esac
    need psql
    url=${HM_ADMIN_URL:-${DATABASE_URL:-}}
    [ -n "$url" ] || die "set HM_ADMIN_URL (or DATABASE_URL) to the table owner's connection string"
    atmp=$(mktemp -d); trap 'rm -rf "$atmp"' EXIT
    case "$sub" in
        add)
            name=$1; role=${2:-}
            if ! r=$(printf "SELECT mem.add_author_role(:'a', NULLIF(:'r', '')::name);\n" \
                     | psql "$url" -X -tA -q -v ON_ERROR_STOP=1 -v a="$name" -v r="$role" \
                       2> "$atmp/err"); then
                die "could not create a login for $name: $(cat "$atmp/err")"
            fi
            cat <<MSG
Login '$r' now reads and writes as '$name'. The database fixes that identity: the mem.reader
setting is ignored for this login, so it cannot read or write as anyone else.

Set its password (prompted; nothing lands in a file or in shell history):

    psql "\${HM_ADMIN_URL:-\$DATABASE_URL}" -c '\\password $r'

Then, on $name's machine and in every job that runs as $name:

    DATABASE_URL=postgresql://$r:<password>@<host>:<port>/<database>
    MEM_AUTHOR=$name     # must match: a write under any other name is refused

and check what the database says from there:

    psql "\$DATABASE_URL" -tAXc 'SELECT * FROM mem.whoami()'     # $r|$name|enforced

Project membership is separate:
    INSERT INTO mem.project_members (author, project) VALUES ('$name', '<tag>');
The boundary holds only while nobody else uses the shared hm_app password (docs/DESIGN.md).
MSG
            ;;
        remove)
            if ! r=$(printf "SELECT mem.remove_author_role(:'r');\n" \
                     | psql "$url" -X -tA -q -v ON_ERROR_STOP=1 -v r="$1" 2> "$atmp/err"); then
                die "could not remove $1: $(cat "$atmp/err")"
            fi
            [ "$r" = t ] || die "$1 is not a mapped author login"
            printf 'Unmapped %s, revoked hm_app from it and disabled its login.\n' "$1"
            ;;
        list)
            psql "$url" -X -v ON_ERROR_STOP=1 \
                -c "SELECT role, author, created_at FROM mem.author_roles ORDER BY author, role"
            ;;
    esac
}

cmd_init() {
    need docker
    env_file="$ROOT/deploy/docker/.env"
    if [ ! -f "$env_file" ]; then
        step "Writing $env_file"
        cp "$ROOT/deploy/docker/.env.example" "$env_file"
        # Generated, not chosen: a password typed in a hurry outlives the hurry.
        # Hex, not base64: a '/' or '+' in the password has to be percent-encoded in the
        # postgresql:// URI this prints, and a password that silently breaks the URI is worse
        # than a shorter one.
        pw=$(openssl rand -hex 16 2>/dev/null | tr -d '\n') \
            || die "could not generate a password -- set POSTGRES_PASSWORD in $env_file yourself"
        [ -n "$pw" ] || die "empty password generated -- set POSTGRES_PASSWORD in $env_file yourself"
        # BSD and GNU sed disagree about -i, so write a new file and move it.
        sed "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$pw|" "$env_file" > "$env_file.tmp"
        mv "$env_file.tmp" "$env_file"
        chmod 600 "$env_file"
    else
        step "Keeping the existing $env_file"     # never clobber a password already in use
    fi

    # shellcheck disable=SC1090
    . "$env_file"
    : "${POSTGRES_USER:=hm}" "${POSTGRES_DB:=hypermnesia}" "${PG_PORT:=5432}"
    [ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is empty in $env_file"

    step "Starting Postgres and the embedder"
    (cd "$ROOT/deploy/docker" && docker compose up -d)

    step "Waiting for Postgres to accept connections"
    i=0
    until docker compose -f "$ROOT/deploy/docker/docker-compose.yml" \
              exec -T postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
        i=$((i + 1))
        [ "$i" -lt 60 ] || die "Postgres did not come up in 60s -- 'docker compose logs postgres'"
        sleep 1
    done

    # Every file is part of the current memory contract. Loading only the base schema leaves the
    # MCP catalog advertising perspectives, evidence and grounded context against tables that do
    # not exist. The owner value is used only to claim unreferenced rows from an older store.
    step "Applying the migrations (./hm migrate, through the container's psql)"
    MIGRATE_VIA=compose
    cmd_migrate --owner "${MEM_AUTHOR:-$(id -un)}"

    url="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:$PG_PORT/$POSTGRES_DB"
    cat <<EOF

Up. Export these, in this shell and in your MCP client's env:

    export DATABASE_URL='$url'
    export EMBED_BACKEND=tei TEI_URL=http://localhost:${TEI_PORT:-8080}

Then put a repo in:  ./hm ingest ~/code/myrepo myrepo
EOF
}

cmd_ingest() {
    # --walk anywhere among the arguments: a notes folder that is git-ignored is the common case
    # for it, and that is exactly the case which otherwise ingests nothing at all.
    #
    # Read with `case` rather than a `[ ]` test, and the positionals rebuilt by assignment rather
    # than by `set -- $(...)`: under `set -e` a failing test as the last statement of a loop ends
    # the script, and an unquoted expansion loses a directory whose name contains a space.
    walk=""; dir=""; scope=""; n=0
    for a in "$@"; do
        case "$a" in
            --walk) walk="--walk" ;;
            *) n=$((n + 1))
               case "$n" in
                   1) dir="$a" ;;
                   2) scope="$a" ;;
               esac ;;
        esac
    done
    [ "$n" -eq 2 ] || usage
    [ -d "$dir" ] || die "no such directory: $dir"
    # Before the first psql call, not inside the python step that runs after it: the scope is
    # interpolated into a single-quoted SQL literal below, so a quote in it produces a syntax
    # error at best and executes as written at worst. ingest_repo.py checks the same alphabet,
    # but by then two queries have already run.
    case "$scope" in
        *[!A-Za-z0-9._-]*|"") die "scope must match [A-Za-z0-9._-]+ (got: $scope)" ;;
    esac
    need psql
    [ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"

    tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT

    # Incremental when the scope already holds documents. The full path OPENS with
    # DELETE FROM documents WHERE repo=<scope>, which cascades to chunks and therefore to every
    # embedding -- a one-file edit costing a full re-embed, quietly. Taking the snapshot from the
    # database we are about to write to is the point: one from anywhere else would mark documents
    # "already stored" that were never there.
    # Do not collapse this into `|| die "cannot reach the store"`: a query that fails for any
    # other reason would then send the reader to debug Postgres connectivity while the store is
    # perfectly healthy.
    if ! n=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
             -c "SELECT count(*) FROM documents WHERE repo='$scope'" 2>&1); then
        die "could not count documents for scope '$scope': $n"
    fi
    if [ "$n" -gt 0 ]; then
        step "Scope '$scope' already holds $n document(s) -- ingesting incrementally"
        psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 -F'	' \
            -c "SELECT path, content_hash FROM documents WHERE repo='$scope'" > "$tmp/known.tsv"
        "$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" \
            --known-hashes "$tmp/known.tsv" $walk
    else
        step "First ingest of scope '$scope'"
        "$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" $walk
    fi

    step "Loading the documents"
    psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f "$tmp/out.sql"

    step "Embedding what has no vector yet"
    # Not fatal here, and not ignored either. The documents are in and the index should still be
    # built, so the run carries on -- but a batch the embedder refused leaves chunks that only the
    # lexical leg can ever find, and the run must end non-zero saying so.
    embed_rc=0
    EMBED_REPO="$scope" "$PY" "$ROOT/ingest/embed_chunks.py" || embed_rc=$?

    # After the rows exist, never before: building HNSW on an empty table then filling it is far
    # slower than the other way round. IF NOT EXISTS makes the re-run a no-op.
    step "Building the ANN index if it is missing"
    psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q \
        -c "CREATE INDEX IF NOT EXISTS chunks_embedding_hnsw ON chunks USING hnsw (embedding vector_cosine_ops)"

    step "Checking the result"
    HM_REPO="$scope" "$PY" "$ROOT/ci/doctor.py" || true
    [ "$embed_rc" -eq 0 ] || die "embedding failed: some chunks of '$scope' were not embedded (embed_chunks exit $embed_rc); they are found by exact words only until './hm ingest' is re-run with the embedder up"
    cat <<EOF

Documents are in. What is NOT in is the part that pays: the Tier 0/1 map -- which files belong to
which component, and which rules must hold for them. Nothing can generate that honestly from a
directory listing. See examples/seed_example.sql and skills/onboard-project.
EOF
}

cmd_refresh() {
    # Every scope, in one pass, from a list -- because the thing that rots a store is not a bad
    # ingest, it is a good one that nobody re-runs. `ci/freshness.py` already counts the drift and
    # writes it down; counting is not fixing, and a number in a log fixes nothing by itself.
    #
    # The list is tab-separated, one scope per line, `#` comments allowed:
    #
    #     /home/me/notes<TAB>notes<TAB>--walk
    #     /home/me/src/api<TAB>api
    #
    # Tabs, not spaces: a directory with a space in its name is ordinary, and splitting on
    # whitespace would quietly ingest the wrong path or nothing at all.
    file=${HM_SCOPES:-$HOME/.config/hypermnesia/scopes}
    [ -f "$file" ] || die "no scope list at $file -- one '<dir><TAB><scope><TAB>[--walk]' per line (or set HM_SCOPES)"
    [ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"
    need psql

    # One refresh at a time. `mkdir` is the atomic test-and-set every shell has. Two overlapping
    # runs would ingest the same scopes concurrently, and the second would delete and rebuild
    # chunks the first is embedding.
    lock="${TMPDIR:-/tmp}/hm-refresh.lock"
    if ! mkdir "$lock" 2>/dev/null; then
        die "another refresh holds $lock"
    fi
    trap 'rmdir "$lock" 2>/dev/null' EXIT INT TERM

    failed=""
    listed=""
    while IFS= read -r line; do
        case "$line" in ''|\#*) continue ;; esac
        dir=$(printf %s "$line" | cut -f1)
        scope=$(printf %s "$line" | cut -f2)
        extra=$(printf %s "$line" | cut -f3)
        [ -n "$scope" ] || die "malformed line in $file (need a TAB between the path and the scope): $line"
        # A scope on the list and not on disk is a misconfiguration, not a reason to skip
        # quietly: the list is hand-written, so a renamed or unmounted directory would drop a
        # whole scope out of every refresh from then on while the run still reported success.
        if [ ! -d "$dir" ]; then
            printf '!! %s: no such directory -- scope %s was NOT refreshed\n' "$dir" "$scope" >&2
            failed="$failed $scope:missing"
            continue
        fi
        step "Refreshing $scope from $dir"
        # In a subshell, because cmd_ingest calls die() on a bad scope or an unreachable store and
        # one bad entry must not take the rest of the list with it.
        #
        # `set +e` around it, and `set -e` INSIDE it, and neither is decoration. A subshell that
        # is the condition of `if` or the left side of `||` has errexit suppressed for everything
        # in it -- and re-setting it inside does not bring it back (checked in dash and in bash's
        # sh mode). Written as `if ! ( cmd_ingest ... )`, the embed step could die with a
        # traceback and the ingest would carry on to build the index and report success. It did:
        # the first run of this command against a live store crashed on a missing psycopg2, said
        # nothing, and was only caught by the unembedded-chunk assertion at the very end.
        set +e
        # shellcheck disable=SC2086  # $extra is one optional flag or nothing, by construction
        ( set -e; cmd_ingest "$dir" "$scope" $extra )
        rc=$?
        set -e
        [ "$rc" -eq 0 ] || failed="$failed $scope"
        listed="${listed:+$listed,}$scope"
    done < "$file"

    # Which scopes does the STORE hold that this list never names? Nothing ties the two together:
    # a directory ingested once by hand and never added to the list is never refreshed again, and
    # search keeps answering about it from whatever it held that day -- with no error, no empty
    # result and no sign that anything is wrong. That is this project's whole subject matter, and
    # the refresh command had the same blind spot it was written to close.
    #
    # A note and not a failure: a scope whose directory is gone for good is a legitimate resident
    # of the store, its documents still searchable history. Failing the run nightly over one would
    # teach whoever reads the output to stop reading it.
    step "Checking the list against the store"
    for r in $(psql "$DATABASE_URL" -tAX -c \
               "SELECT repo FROM documents GROUP BY repo ORDER BY repo" 2>/dev/null); do
        case ",$listed," in *",$r,"*) continue ;; esac
        printf -- '-- %s: in the store, not in %s -- nothing refreshes it\n' "$r" "$file"
    done

    # The check that makes the loop mean something. A chunk with no embedding raises no error
    # anywhere: it is simply absent from the vector half of every search, while the document it
    # belongs to still counts as indexed and the scope still counts as refreshed.
    step "Checking that nothing is left unembedded"
    if ! left=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
                -c "SELECT count(*) FROM chunks WHERE embedding IS NULL" 2>&1); then
        printf '!! could not count unembedded chunks: %s\n' "$left" >&2
        failed="$failed count"
    else
        case "$left" in
            0) printf -- '-- all chunks embedded\n' ;;
            ''|*[!0-9]*) printf '!! unexpected count output: %s\n' "$left" >&2; failed="$failed count" ;;
            *) printf '!! %s chunk(s) left without an embedding\n' "$left" >&2
               failed="$failed embeddings:$left" ;;
        esac
    fi

    [ -z "$failed" ] || die "refresh finished with problems:$failed"
    printf '\nrefresh clean\n'
}

[ $# -ge 1 ] || usage 0
sub=$1; shift
case "$sub" in
    init)   cmd_init "$@" ;;
    ingest) cmd_ingest "$@" ;;
    refresh) cmd_refresh "$@" ;;
    migrate) cmd_migrate "$@" ;;
    author)  cmd_author "$@" ;;
    install) exec "$PY" "$ROOT/ci/install.py" "$@" ;;
    doctor)  exec "$PY" "$ROOT/ci/doctor.py" "$@" ;;
    latency) exec "$PY" "$ROOT/ci/latency.py" "$@" ;;
    -h|--help|help) usage 0 ;;
    *)      die "unknown command: $sub (try ./hm help)" ;;
esac