.PHONY: all bench-all build bump-build check ci-verify clean cli-build compose-validate env-sync doc-health doctor go-build help help-dev help-legacy help-lint help-ops help-scan init install-hooks ja4p-validate link-check lint lint-ansible lint-docs lint-phases lint-opengrep lint-semgrep logs management-down management-logs management-shell management-up rebuild reload remote-bot sbom scan scan-exceptions scorecard-local setup-build start start-poc status stop sync test test-component-suites test-ip test-race test-ratio traffic-off traffic-on tunnel verify-all preflight poc-secrets PYTHON ?= $(shell command -v python || command -v python3 || echo python) # Prefer snap Go if present (local dev), else fall back to env/system GOROOT. GOROOT := $(shell if [ -d /snap/go/current ]; then echo /snap/go/current; elif [ -n "$$GOROOT" ]; then echo "$$GOROOT"; else go env GOROOT 2>/dev/null; fi) GO := $(shell if [ -x /snap/go/current/bin/go ]; then echo GOROOT=$(GOROOT) /snap/go/current/bin/go; elif [ -n "$$GOROOT" ]; then echo GOROOT=$(GOROOT) go; else echo go; fi) # ── Phony targets ───────────────────────────────────────────────────────────── # ── Build Metadata ──────────────────────────────────────────────────────────── VERSION ?= $(shell cat VERSION 2>/dev/null || echo "2.0") BUILD_NUMBER ?= $(shell git rev-list --count HEAD 2>/dev/null || echo "0") GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") FULL_VERSION := v$(VERSION).$(BUILD_NUMBER) TOOLS_IMG := ja4proxy-tools LDFLAGS := -ldflags="-s -w -buildid= -X github.com/seanpor/ja4proxy/internal/config.Version=$(FULL_VERSION) \ -X github.com/seanpor/ja4proxy/internal/config.GitCommit=$(GIT_COMMIT) \ -X github.com/seanpor/ja4proxy/internal/config.BuildDate=$(BUILD_DATE)" # ── Environment Configuration ──────────────────────────────────────────────── ifneq (,$(wildcard .env)) -include .env export endif # Makefile for JA4 Proxy # ── Phony targets # (alphabetical by group) ────────────────────────────────── # Load machine-specific worker count (generated by: make test-calibrate) -include .local/machine.mk # Set reasonable default test worker count (override with WORKERS=N) WORKERS ?= 4 # Normalise NAME= variable — accept Name= or name= as common typos NAME ?= $(or $(Name),$(name)) # Default target all: help doctor: ## Phase 147/225/603 — Verify environment and toolchain health @echo "=== JA4proxy Doctor: Environment Health Check ===" @echo "── Required on the host (build will fail without these) ──────────────" @command -v docker > /dev/null && echo " ✓ docker binary" || { echo " ✗ docker NOT found — required to run containerized tools and build images"; exit 1; } @docker info > /dev/null 2>&1 && echo " ✓ docker daemon responsive" || { echo " ✗ docker daemon NOT running or not accessible by current user"; exit 1; } @$(GO) version 2>/dev/null | grep -E "go1\.(2[6-9]|[3-9][0-9])" > /dev/null && echo " ✓ Go ($$($(GO) version | awk '{print $$3}'))" || { echo " ✗ Go 1.26+ required, found: $$($(GO) version 2>&1)"; exit 1; } @command -v gcc > /dev/null && echo " ✓ C compiler ($$(gcc --version 2>&1 | head -n1 | awk '{print $$1, $$3}'))" || { echo " ✗ gcc NOT found — required for hardened static-pie CGO build mode"; exit 1; } @command -v $(PYTHON) > /dev/null && echo " ✓ python3 ($$($(PYTHON) --version 2>&1 | awk '{print $$2}'))" || { echo " ✗ python3 NOT found"; exit 1; } @echo "── Containerized Toolchain (hermetic; zero host dependencies) ───────" @docker image inspect $(TOOLS_IMG) > /dev/null 2>&1 \ && echo " ✓ $(TOOLS_IMG) image present (Python linters, test runner, pip-audit, doc tools)" \ || echo " · $(TOOLS_IMG) image absent — will build on demand on first 'make lint' / 'make test'" @docker image inspect ja4proxy-bandit > /dev/null 2>&1 \ && echo " ✓ ja4proxy-bandit image present (bandit SAST on Python 3.11)" \ || echo " · ja4proxy-bandit image absent — will build on demand on first 'make lint'" @echo " ✓ Official pinned images used on demand: trivy, hadolint, gitleaks, semgrep, shellcheck," @echo " golangci-lint, checkmake, luacheck, ansible-lint, promtool, amtool, helm, scorecard" @echo "── Environment Configuration ────────────────────────────────────────" @[ -f .env ] && echo " ✓ .env present" || echo " · .env absent — created by 'make start-poc' or 'cp template.env .env'" @echo "=== Doctor: environment and toolchain OK ===" # ── Master help ─────────────────────────────────────────────────────────── help: ## Show the essential front-door targets @echo "======================================================================" @echo " JA4proxy $(FULL_VERSION) — Essential Operational Commands" @echo "======================================================================" @echo "" @echo "── First time? ───────────────────────────────────────────────────────" @echo " make doctor - Check local environment for missing tools" @echo " make init - Run guided setup wizard (generates .env)" @echo "" @echo "── Build ─────────────────────────────────────────────────────────────" @echo " make go-build - Compile Go proxy + CLI binaries (~5s)" @echo " make compose-validate - Validate env vars in compose files (fast)" @echo " make build - go-build + all Docker images (requires .env)" @echo "" @echo "── Operate ───────────────────────────────────────────────────────────" @echo " make start - Start full stack (Proxy + Monitoring)" @echo " make start-poc - Start POC environment only" @echo " make stop - Stop all services" @echo " make status - Show cluster health and security state" @echo " make logs - Stream real-time structured proxy logs" @echo " make reload - Reload config without restart (SIGHUP)" @echo "" @echo "── Quality gates (three tiers) ───────────────────────────────────────" @echo " make check - FAST: compile + env validate + tests (~3m)" @echo " Matches CI required checks. Run before push." @echo " make preflight - FULL: lint + scan + test (~25m, pre-PR)" @echo " Includes Trivy CVE scan — run 'make build' first." @echo " make verify-all - RELEASE: preflight + heavy benchmarks (~45m)" @echo "" @echo "── Incident response ─────────────────────────────────────────────────" @echo " make attack-status - Quick security snapshot (bans, block totals)" @echo " make top-attackers - Top 10 fingerprints by traffic" @echo " make dial LEVEL=75 - Raise blocking threshold (0=monitor, 100=full)" @echo " make block-ip IP=x.x.x.x - Hard-block IP for 1 hour" @echo " make test-ip IP=x.x.x.x - Simulate pipeline decision for an IP" @echo " make help-ops - Full incident response command reference" @echo "" @echo "── More help ─────────────────────────────────────────────────────────" @echo " make help-ops - Incident response + threat-intel details" @echo " make help-lint - Granular linter control" @echo " make help-scan - Granular scanner control" @echo " make help-dev - Build, benchmark, and agent targets" @echo "" @$(MAKE) -s help-lint help-ops: ## Incident response and threat intelligence help @echo "" @echo " Forms being stuffed? Start here:" @echo "" @echo "── Step 1: understand what is happening ──────────────────────────────" @echo " make attack-status - Bans, block totals, top threats" @echo " make top-attackers - Top 10 fingerprints by traffic" @echo " make logs - Live structured log stream" @echo "" @echo "── Step 2: raise the blocking threshold ──────────────────────────────" @echo " make dial LEVEL=50 - Start scoring + light blocking" @echo " make dial LEVEL=75 - Block most bots, pass browsers" @echo " make dial LEVEL=100 - Full blocking at configured thresholds" @echo " (dial=0 is monitor-only; raise gradually to avoid false positives)" @echo "" @echo "── Step 3: targeted blocks (no restart needed) ───────────────────────" @echo " make block-ip IP=1.2.3.4 - Hard-block IP for 1 hour" @echo " make block-ja4 FP=t13d... - Blacklist JA4 fingerprint (instant RST)" @echo " make unblock-ip IP=1.2.3.4 - Remove all blocks for IP (false positive)" @echo " make flush-redis - Nuclear option: reset all bans/blocks/rates" @echo "" @echo "── Step 4: reload without downtime ───────────────────────────────────" @echo " make reload - Push config changes via SIGHUP" @echo " make test-ip IP=1.2.3.4 - Verify decision for a specific IP" @echo "" @echo "── Threat Intelligence ───────────────────────────────────────────────" @echo " make fetch-db - Fetch malicious fingerprints (ja4db/FoxIO)" @echo " make list-pending - Show fingerprints awaiting approval" @echo " make approve-all - Approve all pending fingerprints" @echo " make update-geoip - Download latest GeoIP DB (run monthly)" @echo " make check-geoip - Check age of current GeoIP database" @echo " make geoip-report - Full geo-blocking report" @echo "" help-lint: ## Show linting commands help @echo "" @echo "── Linting ───────────────────────────────────────────────────" @echo " lint-static - Python: mypy + bandit + ruff + pip-audit" @echo " lint-go - Go: go-lint + lint-go-full + lint-go-mod" @echo " go-lint - Go: go vet (fast static analysis)" @echo " lint-shell - shellcheck all .sh scripts" @echo " lint-yaml - yamllint config/ and deploy/monitoring/" @echo " lint-docker - hadolint + docker compose config check" @echo " lint-prom - promtool alert + recording rule validation" @echo " lint-alertmanager - amtool Alertmanager config validation" @echo " lint-secrets - gitleaks scan of git history" @echo " lint-deps - run govulncheck/gosec (via container) + pip-audit" @echo " lint-docs - core doc quality: doc-health + link-check + ATT&CK" @echo " lint-phases - Validate manifest + phase doc integrity" @echo " lint-all - Run every linter in one shot" # ── Scan sub-help ──────────────────────────────────────────────────────── help-scan: ## Show security scanning commands help @echo "" @echo "── Security Scanning ─────────────────────────────────────────" @echo " scan - Run ALL security + container scans (full gate)" @echo " scan-all - Alias of 'scan'" @echo " scan-container - Go SAST (gosec) in a container" @echo " scan-local - gosec locally (view output)" @echo " scan-exceptions - List .trivyignore.first-party / .trivyignore.third-party + days-to-expiry" @echo "" @echo "── Container Scanning ────────────────────────────────────────" @echo " scan-images - Trivy scan of production images" @echo " scan-dockerfiles - Trivy scan of Dockerfiles (IaC)" @echo " scan-first-party - Trivy scan of 1st-party Python/Go deps" @echo " check-image-versions - Check base image tags against Dockerfiles" @echo " check-updates - Check all deps (Docker, Go, Python, Node) for updates" @echo "" @echo "── CVE baseline: PR gate vs scheduled monitor (Phase 847c) ────" @echo " scan-regression - PR gate: scans only the surfaces this branch touches" @echo " scan-regression-plan - Print those surfaces as JSON (no scanning; CI helper)" @echo " scan-baseline - Record security/baseline.json (run on main)" @echo " scan-monitor - Full scan + drift report (APPLY=1 files deduplicated issues)" @echo " medium-inventory - Measure the MEDIUM surface (Phase 847e)" # ── Legacy (Python) sub-help ───────────────────────────────────────────── help-legacy: ## Show legacy Python proxy sub-help @echo "" @echo "── Signal-Score Audit ──────────────────────────────────────" @echo " check-scores - Audit Go/Python scores vs signal registry" # ── Dev sub-help ───────────────────────────────────────────────────────── help-dev: ## Show developer commands sub-help @echo "" @echo "── Build ─────────────────────────────────────────────────────" @echo " build - Build Go binaries + all Docker images" @echo " rebuild - Wipe volumes/images, rebuild, start" @echo " clean - Stop + remove all containers/volumes" @echo "" @echo "── Compose file map ──────────────────────────────────────────" @echo " docker-compose.yml Quick start / CI smoke test" @echo " 3 containers, sensible defaults" @echo " deploy/docker/docker-compose.poc.yml Full local dev (make start)" @echo " HAProxy+proxy+Redis+analytics" @echo " +management. Needs .env" @echo " deploy/docker/docker-compose.prod.yml Single-instance production" @echo " deploy/docker/docker-compose.monitoring.yml Prometheus/Grafana overlay" @echo " deploy/docker/docker-compose.scale.yml 4× proxy workers overlay" @echo " deploy/docker/docker-compose.test.yml Go integration/chaos test env" @echo " deploy/docker/docker-compose.redis-tls.yml Redis-TLS test overlay" @echo "" @echo "── Startup / Shutdown ────────────────────────────────────────" @echo " start - Start full stack (POC + monitoring)" @echo " start-monitoring - Start monitoring stack only" @echo " start-scaled - Start 4-worker scaled + HAProxy" @echo " stop - Stop all services (keep Redis data)" @echo " stop-clean - Stop all + wipe volumes (fresh slate)" @echo " status - Show health + security state" @echo " logs - Stream proxy container logs" @echo " ssh-tunnels - Print SSH tunnel command for remote UI" @echo " remote-bot - Run the TLS test bot: make remote-bot HOST= [PORT=]" @echo "" @echo "── Testing ───────────────────────────────────────────────────" @echo " test - Run all tests locally (fast, no Docker)" @echo " test-unit - Run unit tests only" @echo " test-chaos - Run chaos/resilience tests only" @echo " test-adversarial - Run adversarial/fuzz tests only" @echo " test-calibrate - Benchmark machine, store worker count" @echo " test-docker - Run tests inside Docker (CI env)" @echo " test-component-suites - Run every component test suite" @echo " smoke-test - Quick sanity check" @echo " smoke-docker - Docker Compose lifecycle smoke test" @echo " smoke-k8s - Helm/kind smoke test (skips if absent)" @echo "" @echo "── Go Proxy ──────────────────────────────────────────────────" @echo " go-build - Build Go proxy binary to bin/ja4p" @echo " go-test - Run all Go unit tests" @echo " go-lint - Run go vet on Go code" @echo "" @echo "── Benchmarking ──────────────────────────────────────────────" @echo " bench - Run all Go benchmarks (micro + macro)" @echo " bench-micro - Go internal component micro-benchmarks" @echo " bench-macro - End-to-end load test (requires: make start)" @echo " ARGS= - Extra flags for macro-bench e.g. ARGS='--duration 60'" @echo "" @echo "── Documentation ─────────────────────────────────────────────" @echo " lint-docs - Validate doc frontmatter" @echo " link-check - Check internal Markdown links" @echo " doc-health - Run all doc quality checks" @echo " test-ratio - Show test-to-code ratio" @echo " sync - Sync roadmap from manifest.yaml" @echo "" @echo "── Multi-Agent ───────────────────────────────────────────────" @echo " agent-up NAME=.. - Start isolated agent env" @echo " agent-down [NAME=] - Stop agent env" @echo " agent-status - List running agent environments" @echo " tunnel NAME=.. - SSH tunnels for agent UIs" @echo "" @echo " After agent-up, these auto-target active agent:" @echo " stop stop-clean status logs health-check" @echo " flush-redis clean rebuild" @echo " management-up/down/logs/shell" @echo "" # ── Startup / Shutdown ──────────────────────────────────────────────────────── # Start full stack (POC + monitoring) start: ## Start full stack (POC + Prometheus/Grafana) @./scripts/start-all.sh # Start monitoring stack only (Prometheus / Grafana / Loki) start-monitoring: ## Start monitoring stack only @./scripts/start-monitoring.sh # Start scaled configuration with 4 workers and HAProxy (Phase 26d) start-scaled: ## Start 4-worker scaled config with HAProxy @echo "Starting JA4Proxy with 4-worker scaling..." @docker compose -f deploy/docker/docker-compose.poc.yml -f deploy/docker/docker-compose.scale.yml --env-file .env up -d @echo "✓ HAProxy load balancer started on port 443" @echo "✓ 4 worker processes started on ports 8080, 8083, 8084, 8085" @echo "✓ HAProxy stats available at http://localhost:8404/stats (admin/admin123)" # Stop all services (keep Redis data) stop: ## Stop all services (keep Redis data) @./scripts/stop-all.sh # Stop all services AND remove all volumes (fresh slate) stop-clean: ## Stop all + wipe volumes (fresh slate) @./scripts/stop-all.sh --clean # Show health of all services + security state status: ## Show health of all services + security state @./scripts/status.sh # ── Remote Manual Testing (Phase 220) ───────────────────────────────────────── # Remote access needs no special stack: the poc + monitoring compose files # already bind every service to AGENT_BIND_IP (default 127.0.0.1), and # start-poc.sh validates it via check_bind_address.py. To test from a laptop: # set AGENT_BIND_IP= in .env, run `make start`, then the # test bot below. See docs/runbooks/REMOTE_TESTING.md. # # Run the TLS test bot against a (remote) proxy. Usage: make remote-bot HOST=10.0.0.5 PORT=443 remote-bot: ## Run test bot against remote proxy (HOST=... PORT=...) @[ -n "$(HOST)" ] || { echo "Usage: make remote-bot HOST= [PORT=]"; exit 1; } @PYTHONPATH=. $(PYTHON) scripts/test-bot.py --proxy "$(HOST)" --port $(or $(PORT),443) # ── Multi-Agent ─────────────────────────────────────────────────────────────── # Start an isolated agent environment (auto-generates .env. if missing). # Writes .current-agent so subsequent commands default to this agent. # Usage: make agent-up NAME=claude agent-up: ## Start an isolated agent environment (NAME=) @[ -n "$(NAME)" ] || (echo "Usage: make agent-up NAME= (agents: gemini|claude|ollama|mistral)"; exit 1) @[ -f ".env.$(NAME)" ] || (echo "→ No .env.$(NAME) found — generating..."; ./scripts/agent-env.sh $(NAME)) docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(NAME) --env-file .env.$(NAME) up -d @echo "$(NAME)" > .current-agent @echo "✓ Agent $(NAME) started (saved to .current-agent)" @IP=$$(grep '^AGENT_BIND_IP=' .env.$(NAME) | cut -d= -f2); \ MGMT=$$(grep '^HOST_PORT_MANAGEMENT=' .env.$(NAME) | cut -d= -f2); MGMT=$${MGMT:-8090}; \ ANL=$$(grep '^HOST_PORT_ANALYTICS=' .env.$(NAME) | cut -d= -f2); ANL=$${ANL:-8080}; \ echo " Ingress: https://$$IP:443"; \ echo " Analytics: http://$$IP:$$ANL"; \ echo " Management:http://$$IP:$$MGMT"; \ echo " Metrics: http://$$IP:9090/metrics" @echo " Admin: ./scripts/ja4-admin.sh status (uses .current-agent automatically)" @echo " Tunnels: make tunnel NAME=$(NAME) HOST=user@this-server" @echo "" @echo "Agent-aware targets (all read .current-agent automatically):" @echo " make stop / stop-clean / status / logs / health-check" @echo " make flush-redis / clean / rebuild" @echo " make management-up/down/logs/shell" # Stop an isolated agent environment. # If NAME is not given, reads .current-agent (set by the last agent-up). # Clears .current-agent if it matches the stopped agent. # Usage: make agent-down NAME=claude OR make agent-down agent-down: ## Stop an isolated agent environment (NAME=) $(eval _NAME := $(or $(NAME),$(shell cat .current-agent 2>/dev/null))) @[ -n "$(_NAME)" ] || (echo "Usage: make agent-down NAME= (or run make agent-up first to set .current-agent)"; exit 1) @[ -f ".env.$(_NAME)" ] || (echo "No .env.$(_NAME) — is agent $(_NAME) configured?"; exit 1) docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_NAME) --env-file .env.$(_NAME) down @if [ "$$(cat .current-agent 2>/dev/null)" = "$(_NAME)" ]; then rm -f .current-agent; echo "✓ Cleared .current-agent"; fi # List all running agent environments and show which is current. agent-status: ## List all running agent environments @echo "Running ja4_* agent environments:" @docker compose ls 2>/dev/null | grep '^ja4_' || echo " (none running)" @if [ -f .current-agent ]; then echo "Current (.current-agent): $$(cat .current-agent)"; fi # ── Build ────────────────────────────────────────────────────────────────────── # Build Docker images build: bump-build ## Build all Docker images (Go compiled inside Docker — no local Go required) @$(MAKE) env-sync @$(MAKE) compose-validate @echo "Building Docker images (BuildKit enabled)..." DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 \ VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env build build-native: bump-build ## Build host-native Go binaries (requires local Go 1.26+) @mkdir -p bin @$(MAKE) go-build cli-build @$(MAKE) env-sync @$(MAKE) compose-validate @echo "Building Docker images (BuildKit enabled)..." DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 \ VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env build # Validate compose files without building images (fast — no Docker daemon build step) # Catches missing required env vars before you waste time on a full 'make build'. # Safe to run without a running Docker daemon; used by CI as a merge gate. env-sync: ## Add any newly-required vars to an existing .env (idempotent, never overwrites) $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @ENVFILE=$$([ -n "$(_AGENT)" ] && echo ".env.$(_AGENT)" || echo ".env"); \ ./scripts/env-sync.sh "$$ENVFILE" compose-validate: ## Validate docker-compose files and required env vars (fast, no image build) @if [ ! -f .env ]; then \ echo "ERROR: .env not found. Run 'make init' to generate it."; \ exit 1; \ fi @docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env config > /dev/null \ || { echo ""; \ echo "If the error above says a variable 'is missing a value', your .env"; \ echo "predates it — start-poc.sh only generates a .env when none exists,"; \ echo "so an existing checkout never picks up newly-required vars."; \ echo "Fix: make env-sync"; \ exit 1; } @echo "✓ compose config valid — all required env vars present" # Run all tests locally in parallel (fast — no Docker required) # Skips tests marked @pytest.mark.live_services (require Go/Python proxy + Redis stack) test: tools-image cli-build ## Phase 146 — Run the full test suite @echo "=== Running Go Native Tests ===" @if command -v go >/dev/null 2>&1 || [ -x /snap/go/current/bin/go ]; then \ GOROOT=$(GOROOT) $(GO) test -v -coverprofile=coverage.txt -covermode=atomic ./...; \ else \ docker run --rm -v "$$PWD":/src -w /src golang:1.27.3-alpine go test -v -coverprofile=coverage.txt -covermode=atomic ./...; \ fi @echo "=== Running Python Unit Tests (containerized: $(TOOLS_IMG)) ===" @$(TOOLS_RUN) pytest tests/unit/ -n $(WORKERS) --dist=loadfile --timeout=60 --tb=short @echo "=== Running management/tests/ (phase-808 — was silently excluded, see manifest) ===" @$(TOOLS_RUN) pytest management/tests/ -n $(WORKERS) --dist=loadfile --timeout=60 --tb=short @echo "=== Running Integration Smoke Tests (containerized) ===" @$(TOOLS_RUN) pytest tests/integration/ -k "not docker_stack" -x -q --timeout=60 @echo "=== Running CI/workflow guardrail tests (containerized) ===" @$(TOOLS_RUN) pytest tests/test_workflow_pinning.py -q --timeout=60 @echo "✓ Full test suite passed" @$(TOOLS_RUN) python scripts/pipeline_summary.py test # Run unit tests only (containerized — pinned Python 3.14, no host venv) test-unit: tools-image ## Run unit tests only @$(TOOLS_RUN) pytest tests/unit/ -n $(WORKERS) --dist=loadfile --timeout=60 --tb=short $(ARGS) @$(TOOLS_RUN) pytest management/tests/ -n $(WORKERS) --dist=loadfile --timeout=60 --tb=short $(ARGS) # Run chaos/resilience tests only (containerized) test-chaos: tools-image ## Run chaos/resilience tests only @$(TOOLS_RUN) pytest tests/chaos/ -n $(WORKERS) --dist=loadfile --timeout=60 --tb=short $(ARGS) # Run adversarial/fuzz tests only (containerized; no parallelism — some are stateful) test-adversarial: tools-image ## Run adversarial/fuzz tests only @$(TOOLS_RUN) pytest tests/adversarial/ --timeout=60 --tb=short $(ARGS) # Run multi-core concurrent preemption stress tests (Go -race with elevated GOMAXPROCS) test-race-stress: ## Run Go concurrency stress harness (Phase 842) @./scripts/run-concurrency-stress.sh # Benchmark this machine and store worker count in .local/machine.mk test-calibrate: ## Benchmark this machine, store worker count @$(PYTHON) scripts/detect_workers.py # Run tests inside Docker (CI / clean environment) test-docker: ## Run tests inside Docker (CI env) @./scripts/run-tests.sh # Run quick smoke test smoke-test: ## Quick sanity check @./scripts/smoke-test.sh # Run linting lint: ## Phase 146 — Run all linters (Python, Go, Infra, Docs) @$(MAKE) lint-all @python3 scripts/pipeline_summary.py lint @python3 scripts/ci_summary.py lint # Security scanning with bandit (medium/high severity, skip B104 bind-all) lint-security: bandit-image ## bandit SAST (medium/high severity) @$(BANDIT_RUN) bandit -r src/analytics/ -ll --skip B104 && echo " ✓ lint-security passed" # Type checking with mypy (suppress output) lint-types: ## Run mypy type checker on src/ docker run --rm -v $(PWD):/app python:3.14.0-slim sh -c "cd /app && pip install mypy && mypy src/ 2>/dev/null || echo 'Mypy warnings above, see baseline docs/reports/MYPY_BASELINE.md'" # Phase 16f static analysis gates — runs locally (no Docker required). # All three tools must pass cleanly; failures block the CI build. # mypy: type-check src/ and proxy.py using mypy.ini baseline # bandit: SAST scan, medium/high severity only (-ll), ignore B104 (bind 0.0.0.0 intentional) # ruff: fast linter/formatter-check (replaces flake8/isort for new code) # pip-audit: CVE scan of requirements.txt; urllib3 CVEs acknowledged (transitive, tracked backlog) lint-static: tools-image bandit-image ## mypy + bandit + ruff + pip-audit @echo "=== mypy: type checking ===" @$(TOOLS_RUN) mypy src/analytics/ && echo " ✓ mypy passed" @echo "" @echo "=== bandit: SAST (medium/high, Python 3.11 — see Dockerfile.bandit) ===" @$(BANDIT_RUN) bandit -r src/analytics/ -ll -q --skip B104 && echo " ✓ bandit passed" @echo "" @echo "=== ruff: linting ===" @$(TOOLS_RUN) ruff check src/analytics/ && echo " ✓ ruff passed (tests advisory only)" @echo "" @echo "=== pip-audit: CVE dependency scan ===" @# phase-311: resilient wrapper — retries transient PyPI/OSV outages and @# soft-passes (with a CI warning) if the service is unreachable, but still @# fails on a real vulnerability. Stops upstream-outage flakes reddening the gate. @# phase-847a: waived CVEs live in security/waivers.yaml and are rendered into @# the ignorefiles by scripts/render_waivers.py, so this gate and the ci.yml @# dependency-audit-python job can no longer drift apart. The old inline list @# here (6 entries) had already diverged from ci.yml's (2 entries). @# Fail fast if a derived file is stale or a hand edit slipped in (the pytest @# drift test covers the same ground; this just fails earlier in the gate). @$(TOOLS_RUN) python scripts/render_waivers.py --check @test -s .pip-audit-ignores || { echo "✗ .pip-audit-ignores missing — run: python3 scripts/render_waivers.py"; exit 1; } @# phase-847f: the registry is coherent — but is it still TRUE? Every waiver's @# reachability premise is bound to a check and executed here, so a service that @# gained a published port (or a dev dependency promoted into a runtime image) @# fails the gate and demands re-justification instead of drifting silently. @$(TOOLS_RUN) python scripts/reachability_checks.py >/dev/null && echo " ✓ waiver reachability premises hold" @$(TOOLS_RUN) bash scripts/pip-audit-resilient.sh -r requirements.txt \ $$(grep -v '^[[:space:]]*#' .pip-audit-ignores) \ && echo " ✓ pip-audit passed (waived CVEs: see security/waivers.yaml)" @echo "" @echo "✓ All static analysis gates passed" # Code quality with flake8 (suppress output) lint-quality: ## flake8 code quality docker run --rm -v $(PWD):/app python:3.14.0-slim sh -c "cd /app && pip install flake8 && flake8 src/analytics/ scripts/ tests/ 2>/dev/null || echo 'Flake8 warnings above, see baseline docs/QUICK_REFERENCE.md'" # Coverage reporting with pytest-cov (Phase 16c gate: ≥80% all modules) lint-coverage: ## pytest-cov coverage reporting (≥80% gate) $(PYTHON) -m pytest tests/ --ignore=tests/integration/test_docker_stack.py \ --cov=src --cov=proxy \ --cov-fail-under=80 \ --cov-report=term-missing \ --cov-report=html:reports/coverage/html # Lint Dockerfiles (hadolint) and validate docker-compose files (docker compose config). # Ignored rules are consciously accepted — see .hadolint.yaml for rationale. # deploy/docker/docker-compose.test.yml is the full Go test environment. # Ignored rules live in .hadolint.yaml (with per-rule rationale), NOT in flags # here. Phase 800: they used to be duplicated as `--ignore` flags because the # container had no volume mount, so .hadolint.yaml could never be read — and the # file additionally used the wrong top-level key (`ignore:` instead of # `ignored:`), which hadolint accepts silently while applying nothing. Both are # fixed; the mount + --config below makes .hadolint.yaml the single source of # truth. Do not reintroduce --ignore flags: they would drift from the file. HADOLINT_DOCKERFILES := deploy/docker/Dockerfile.management \ deploy/docker/Dockerfile.mockbackend deploy/docker/Dockerfile.test \ deploy/docker/Dockerfile.trafficgen deploy/docker/Dockerfile.go-proxy deploy/docker/Dockerfile.ja4-tap \ src/analytics/Dockerfile src/tarpit/Dockerfile \ tests/docker/Dockerfile.recorder \ tests/docker/Dockerfile.test-runner tests/docker/Dockerfile.tls-backend lint-docker: ## hadolint + `docker compose config --quiet` (all overlays) @echo "=== hadolint: Dockerfiles ===" @# Pre-pull so Docker's "Unable to find image / Downloaded" messages (stderr, @# emitted on a cold runner) don't land in $$result below and false-FAIL the @# first file. (We keep 2>&1 to still capture genuine hadolint errors.) @docker pull -q hadolint/hadolint:v2.14.0 >/dev/null 2>&1 || true @for f in $(HADOLINT_DOCKERFILES); do \ printf " %-50s" "$$f"; \ result=$$(docker run --rm -i -v "$(PWD):/src:ro" -w /src hadolint/hadolint:v2.14.0 hadolint --config .hadolint.yaml --no-color - < "$$f" 2>&1); \ if [ -n "$$result" ]; then echo "FAIL"; echo "$$result"; exit 1; fi; \ echo "OK"; \ done @echo "" @echo "=== docker compose config: compose files ===" @# Phase-202: the `:?` required-credential syntax in poc/monitoring compose @# files forces operators to supply real secrets. For `config --quiet` lint @# we inject harmless placeholders so the static validation still runs. @docker compose -f deploy/docker/docker-compose.test.yml config --quiet \ && echo " deploy/docker/docker-compose.test.yml OK" @REDIS_PASSWORD=lint-placeholder GRAFANA_PASSWORD=lint-placeholder \ HAPROXY_STATS_USER=lint-placeholder HAPROXY_STATS_PASSWORD=lint-placeholder \ EXPORTER_REDIS_PASSWORD=lint-placeholder \ docker compose -f deploy/docker/docker-compose.monitoring.yml config --quiet \ && echo " deploy/docker/docker-compose.monitoring.yml OK" @BACKEND_HOST=lint-placeholder ANALYTICS_REDIS_PASSWORD=lint-placeholder TAP_INTERFACE=lint-placeholder \ docker compose -f deploy/docker/docker-compose.prod.yml config --quiet \ && echo " deploy/docker/docker-compose.prod.yml OK" @BACKEND_HOST=lint-placeholder REDIS_PASSWORD=lint-placeholder ANALYTICS_REDIS_PASSWORD=lint-placeholder \ MANAGEMENT_REDIS_PASSWORD=lint-placeholder ANALYTICS_HMAC_SECRET=lint-placeholder \ MANAGEMENT_JWT_SECRET=lint-placeholder MANAGEMENT_ADMIN_USER=lint-placeholder MANAGEMENT_ADMIN_PASSWORD=lint-placeholder \ docker compose -f deploy/docker/docker-compose.poc.yml -f deploy/docker/docker-compose.scale.yml config --quiet \ && echo " deploy/docker/docker-compose.scale.yml (overlay) OK" @echo "" @echo "✓ Docker lint passed" # Pinned lint toolchain (Phase 313). The Python linters + project Python scripts # run inside $(TOOLS_IMG) (built from Dockerfile.tools) via $(TOOLS_RUN), so CI # and local share one reproducible env with no host `pip install`. Tools with # their own official images (hadolint, shellcheck, trivy, gitleaks, gosec, # promtool, amtool, scorecard, lychee, semgrep) are run from those directly. tools-image: ## Build containerized tools image (Dockerfile.tools) @docker build -q -t $(TOOLS_IMG) -f Dockerfile.tools . >/dev/null # Docker socket mounted so tests/integration/test_container_config.py, # test_dockerfile_coverage.py, and test_go_proxy_image.py (all gated on # `shutil.which("docker")` + a reachable daemon) actually run instead of # silently skipping (phase-514). The tools image runs as root (no USER # directive), so no socket-permission workaround is needed. TOOLS_RUN = docker run --rm -v $(PWD):/src -w /src -v /var/run/docker.sock:/var/run/docker.sock $(TOOLS_IMG) # Bandit runs on Python 3.11 (see Dockerfile.bandit): its B502/B604/B607 plugins # use ast.Num, removed in 3.12, which makes bandit silently skip whole files on # the 3.14 tools image. Separate image keeps full SAST coverage. BANDIT_IMG := ja4proxy-bandit bandit-image: ## Build containerized bandit SAST image (Dockerfile.bandit) @docker build -q -t $(BANDIT_IMG) -f Dockerfile.bandit . >/dev/null BANDIT_RUN = docker run --rm -v $(PWD):/src -w /src $(BANDIT_IMG) # Opengrep SAST toolchain (Phase 865: replaces Semgrep with pinned binary & local rules) OPENGREP_IMG := ja4proxy-opengrep:1.33.1 opengrep-image: ## Build containerized Opengrep SAST image @if [ -z "$$(docker images -q $(OPENGREP_IMG) 2>/dev/null)" ]; then \ echo "Building $(OPENGREP_IMG)..."; \ docker build -t $(OPENGREP_IMG) -f deploy/docker/Dockerfile.opengrep deploy/docker/ >/dev/null; \ fi SEMGREP_IMG := $(OPENGREP_IMG) # checkmake publishes no semantic version tags — only `latest` and per-commit # tags, all from 2022-02-18. Pinned to the newest immutable commit tag rather # than `latest` (digest sha256:eb6919b20b22d1701a976856e4a224627df0a74b118246101fb6cf5c2e03049f). # The image's ENTRYPOINT is `./checkmake /Makefile`, which ignores our args, so # the recipe overrides it with --entrypoint /checkmake. CHECKMAKE_IMG := mrtazz/checkmake:2c59d1f0939900ca3a4208fb9b9300de90ecee8a # Lint shell scripts with shellcheck (error-level only; warnings are advisory). # SC2154 suppressed: variables sourced from .env are referenced but not assigned in-script. # `.local/` holds container-written caches (Trivy's fanal DB among them) owned # by root, which this find cannot traverse. It was printing a "Permission # denied" warning on EVERY make invocation — this variable is expanded at # parse time, so the noise appeared before any target ran and looked like a # failure of whatever target was asked for. Nothing there is a shell script. SHELL_SCRIPTS := $(shell find . \( -path "./.git" -o -path "./.local" -o -path "./node_modules" -o -path "./.claude" \) -prune -o -name "*.sh" -print | sort) lint-shell: ## shellcheck all `.sh` scripts (error-level) @echo "=== shellcheck: shell scripts ===" @fail=0; \ for f in $(SHELL_SCRIPTS); do \ printf " %-60s" "$$f"; \ result=$$(docker run --rm -i koalaman/shellcheck:v0.11.0 \ --severity=error --exclude=SC2154 - < "$$f" 2>/dev/null); \ if [ -n "$$result" ]; then echo "FAIL"; echo "$$result"; fail=1; \ else echo "OK"; fi; \ done; \ [ $$fail -eq 0 ] || exit 1 @echo "" @echo "✓ Shell lint passed" # Scan third-party Docker images for HIGH/CRITICAL CVEs (Trivy). # Third-party image list for CVE scanning. Keep in sync with docs/reference/DOCKER_IMAGES.md. # Fails only on CRITICAL findings; HIGH findings are reported but non-blocking. # Scans only pinned third-party images (not images we build ourselves — use scan-first-party for those). # Shared Trivy cache (Phase 227): mounted into every trivy container so the # vulnerability DB is downloaded ONCE per `make scan` instead of re-fetched for # each image. In CI this dir is also restored/saved via actions/cache. TRIVY_CACHE ?= $(HOME)/.cache/trivy # Covers images deployed in deploy/docker/docker-compose.prod.yml + deploy/docker/docker-compose.monitoring.yml. # Phase 810: derived from those compose files via # scripts/check_image_versions.py --list-third-party instead of hand-copied # here — a hand-maintained duplicate is exactly how this list went stale # (one image's tag) and silently incomplete (three images added to # docker-compose.monitoring.yml over time were never added here, so they went # unscanned indefinitely). The compose files are now the only place a # third-party image version is written down. Lazily-expanded (`=`, not `:=`) # so the script only runs when a recipe references $(TRIVY_IMAGES), not on # every `make` invocation. See scan-images below for the failure-mode guard # (an empty or error'd list must not silently scan zero images). TRIVY_IMAGES = $(shell $(PYTHON) scripts/check_image_versions.py --list-third-party) # Manifest consistency check. Verifies: TODO.md + PROJECT_STATUS.md generate # cleanly from manifest.yaml; every COMPLETE phase has a CHANGELOG entry (or # is in the frozen HISTORICAL_CHANGELOG_GAPS baseline — see the script). # Phase 800: wired into lint-docs-all (below) so this is a real, CI-enforced # gate, not an easy-to-forget manual step before a phase close-out. It had # drifted to 544 permanent failures (240 CHANGELOG + a since-retired, # CLAUDE.md-contradicting 304-failure TABLE check) before nobody running it # regularly let that happen — see the script's own docstring. check-manifest: ## Verify manifest.yaml / TODO.md / CHANGELOG.md stay consistent @$(PYTHON) scripts/check_manifest.py scan-images: ## Trivy scan of third-party images (fails on HIGH or CRITICAL) @mkdir -p "$(TRIVY_CACHE)" @echo "=== Trivy: third-party image CVE scan (HIGH + CRITICAL) ===" @echo " Fails on HIGH or CRITICAL. Waivers are authored in security/waivers.yaml" @echo " and generated into .trivyignore.third-party by scripts/render_waivers.py." @echo "" @images="$$($(PYTHON) scripts/check_image_versions.py --list-third-party)" || { \ echo "✗ Could not derive the third-party image list from the compose"; \ echo " files (version drift between them?) — fix that before scanning."; \ exit 1; \ }; \ [ -n "$$images" ] || { echo "✗ scripts/check_image_versions.py --list-third-party returned no images — check compose file paths"; exit 1; }; \ fail=0; \ for img in $$images; do \ echo " Scanning $$img ..."; \ TRIVY_CACHE="$(TRIVY_CACHE)" $(PYTHON) scripts/trivy_image_report.py --ignorefile /scan/.trivyignore.third-party "$$img" || fail=1; \ done; \ [ $$fail -eq 0 ] || { \ echo "✗ HIGH/CRITICAL CVEs found, or a scan could not be run — Phase 847b: a"; \ echo " scan that fails to produce JSON is treated as a FAILURE, never a pass."; \ echo " Waivers are authored in security/waivers.yaml and generated into"; \ echo " .trivyignore.third-party by scripts/render_waivers.py — see"; \ echo " docs/runbooks/security_scan_exceptions.md"; \ exit 1; \ }; \ echo "✓ Image scan complete — no findings" # Trivy misconfiguration scan of Dockerfiles and compose files. # Does NOT require building images — analyses file content only. # Catches: running as root, missing HEALTHCHECK, ADD with URL, exposed secrets, missing no-new-privileges. # Policy: HIGH + CRITICAL → exit 1 (we own these files; zero tolerance). scan-dockerfiles: ## Trivy config scan of Dockerfiles + compose files (HIGH/CRITICAL → fail) @mkdir -p "$(TRIVY_CACHE)" @echo "=== Trivy: Dockerfile/compose misconfiguration scan (HIGH + CRITICAL) ===" @echo " Fails on HIGH or CRITICAL findings." @fail=0; \ for f in deploy/docker/Dockerfile.go-proxy deploy/docker/Dockerfile.management \ deploy/docker/Dockerfile.mockbackend deploy/docker/Dockerfile.test \ deploy/docker/Dockerfile.trafficgen deploy/docker/Dockerfile.cli \ deploy/docker/Dockerfile.ja4-tap \ ; do \ if [ -f "$$f" ]; then \ echo ""; \ echo " ── Scanning $$f ──"; \ docker run --rm \ -v "$(PWD):/scan:ro" \ -v "$(TRIVY_CACHE):/root/.cache/trivy" \ aquasec/trivy:0.71.0 config \ --severity HIGH,CRITICAL --exit-code 1 \ --skip-version-check \ "/scan/$$f" || fail=1; \ fi; \ done; \ [ $$fail -eq 0 ] || { echo ""; echo "✗ Dockerfile scan failed — fix HIGH/CRITICAL findings above"; exit 1; } @echo "" @echo "✓ All Dockerfiles passed" @echo " (compose files validated separately via 'docker compose config' in CI)" # Build every first-party image the CVE gates cover. Split out of # `scan-first-party` (Phase 847c) so the PR regression gate's CI job can build # exactly the prerequisites it needs — including the profile-gated # test/trafficgen/tap images that 'make build' skips — without duplicating these # build lines into workflow YAML, where they would silently drift from this file. # Builds every image `scan-first-party` / the baseline model scans. # # Phase 847c follow-up (found by the first CI run of scan-baseline-monitor.yml): # the five `make build`-produced images used to be built only when NOT in CI # (`[ -z "$CI" ]`), because the one CI caller at the time — the `Security Scan` # job — had already built them via `make build`, and a --no-cache rebuild of # five images is minutes of redundant CI time. But 847c added two callers that # do NOT run `make build` first: the monitor workflow, and the scan-regression # PR gate when a first-party surface is in scope. Under the old guard both got # "unable to find the specified image" and exit 2 — fail-closed, but a monitor # that can never scan and a PR gate that reddens any PR touching a first-party # Dockerfile. The guard now keys on *presence in this job* instead of on `$CI`: # local runs still rebuild --no-cache (Phase 800's stale-layer-cache rationale), # the Security Scan job still pays nothing (the images are present), and any # other caller — CI or local — gets the images it needs. build-first-party-images: @if [ -z "$$CI" ]; then \ echo " Rebuilding all first-party images with --no-cache (local run)..."; \ DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.go-proxy -t ja4proxy:2.0.0 . >/dev/null; \ DOCKER_BUILDKIT=1 docker build --no-cache -q -f src/analytics/Dockerfile -t ja4proxy-analytics:1.0.0 . >/dev/null; \ DOCKER_BUILDKIT=1 docker build --no-cache -q -f src/tarpit/Dockerfile -t ja4proxy-tarpit:1.0.0 src/tarpit >/dev/null; \ DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.mockbackend -t ja4proxy-mockbackend:1.0.0 . >/dev/null; \ DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.management -t ja4proxy-management:1.0.0 . >/dev/null; \ else \ echo " CI: ensuring each first-party image exists in this job (no rebuild if present)..."; \ for spec in \ "deploy/docker/Dockerfile.go-proxy ja4proxy:2.0.0 ." \ "src/analytics/Dockerfile ja4proxy-analytics:1.0.0 ." \ "src/tarpit/Dockerfile ja4proxy-tarpit:1.0.0 src/tarpit" \ "deploy/docker/Dockerfile.mockbackend ja4proxy-mockbackend:1.0.0 ." \ "deploy/docker/Dockerfile.management ja4proxy-management:1.0.0 ."; do \ set -- $$spec; df=$$1; tag=$$2; ctx=$$3; \ if docker image inspect "$$tag" >/dev/null 2>&1; then \ echo " $$tag: already present, not rebuilding"; \ else \ DOCKER_BUILDKIT=1 docker build -q -f "$$df" -t "$$tag" "$$ctx" >/dev/null; \ fi; \ done; \ fi @echo " Building the profile-gated CI-only test/trafficgen images (skipped by 'make build')..." @DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.test -t ja4proxy-test:1.0.0 . >/dev/null @DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.trafficgen -t ja4proxy-trafficgen:1.0.0 . >/dev/null @echo " Building ja4proxy-tap (deploy/docker/docker-compose.prod.yml's 'tap' profile, skipped by 'make build')..." @DOCKER_BUILDKIT=1 docker build --no-cache -q -f deploy/docker/Dockerfile.ja4-tap -t ja4proxy-tap:1.0.0 . >/dev/null # CVE scan of first-party images we build. # Policy (Phase 317): HIGH + CRITICAL → exit 1. We build/own these images, so a # HIGH is actionable (base bump / dep bump / dated .trivyignore.first-party). The profile-gated # CI-only test + trafficgen images are skipped by 'make build', so this target # builds them explicitly — otherwise they would be scanned-as-absent (a vacuous pass). scan-first-party: build-first-party-images ## Trivy CVE scan of built images (fails on HIGH or CRITICAL) @mkdir -p "$(TRIVY_CACHE)" @echo "=== Trivy: first-party image CVE scan (HIGH + CRITICAL) ===" @echo " Phase 317: fails on HIGH or CRITICAL. .trivyignore.first-party holds dated exceptions." @echo " Phase 800: a cached apk/apt layer can sit for weeks and silently hide a" @echo " since-fixed OS CVE (observed: ja4proxy-test carried a 5-week-stale c-ares" @echo " layer that a plain 'make build' never re-executed). CI runners are always" @echo " cache-free, so this only bites local runs — rebuild --no-cache here to match." @echo "" @fail=0; \ for img in ja4proxy:2.0.0 ja4proxy-analytics:1.0.0 ja4proxy-tarpit:1.0.0 ja4proxy-mockbackend:1.0.0 ja4proxy-management:1.0.0 ja4proxy-test:1.0.0 ja4proxy-trafficgen:1.0.0 ja4proxy-tap:1.0.0; do \ echo " Scanning $$img ..."; \ TRIVY_CACHE="$(TRIVY_CACHE)" $(PYTHON) scripts/trivy_image_report.py \ --ignorefile /scan/.trivyignore.first-party --docker-socket "$$img" || fail=1; \ done; \ [ $$fail -eq 0 ] || { \ echo "✗ HIGH/CRITICAL CVEs found, or a scan could not be run — Phase 847b:"; \ echo " a scan that fails to produce JSON is a FAILURE, never a pass."; \ echo " Fix it by bumping the base image or dependency, or add a justified,"; \ echo " dated .trivyignore.first-party entry — authored in"; \ echo " security/waivers.yaml (scripts/render_waivers.py), see"; \ echo " docs/runbooks/security_scan_exceptions.md"; \ exit 1; \ } @echo "✓ First-party image scan complete (HIGH + CRITICAL gate)" # Phase 228: human-readable rollup of the image CVE scans. Reporting only — # `make scan` remains the authoritative gate. First-party rows require `make build`. FIRST_PARTY_IMAGES := ja4proxy:2.0.0 ja4proxy-analytics:1.0.0 ja4proxy-tarpit:1.0.0 ja4proxy-mockbackend:1.0.0 ja4proxy-management:1.0.0 ja4proxy-test:1.0.0 ja4proxy-trafficgen:1.0.0 ja4proxy-tap:1.0.0 scan-summary: ## Phase 228 — compact CRIT/HIGH/MED rollup of all scans (images + misconfig + gosec; reporting only) @mkdir -p "$(TRIVY_CACHE)" @# Phase 847a: pass the correct scope-split ignorefile (Phase 822 deleted the @# single .trivyignore this used to hard-code, so waivers had stopped applying). @$(PYTHON) scripts/scan_summary.py --ignorefile .trivyignore.third-party $(TRIVY_IMAGES) @$(PYTHON) scripts/scan_summary.py --ignorefile .trivyignore.first-party $(FIRST_PARTY_IMAGES) @# gosec rollup (best-effort; builds the pinned security-scan image) @docker build -q -t ja4proxy-security-scan -f deploy/docker/security-scan/Dockerfile . >/dev/null 2>&1 \ && docker run --rm -v "$(PWD):/app" ja4proxy-security-scan \ "gosec -fmt=json -exclude-dir=.claude ./... 2>/dev/null" \ | $(PYTHON) scripts/scan_summary.py gosec \ || echo " ! gosec summary skipped (build/docker unavailable)" scan-exceptions: ## List .trivyignore.first-party / .trivyignore.third-party exceptions with days-to-expiry @$(PYTHON) scripts/scan_exceptions.py # Phase 847c: the PR regression gate. Scans ONLY the surfaces this branch # touches and fails only on findings absent from security/baseline.json, so an # upstream CVE disclosed overnight cannot redden an unrelated PR — while a PR # that bumps an image or a dependency still gets that surface scanned. BASE_REF ?= origin/main scan-regression: ## PR gate: scan only the surfaces this branch touches (fails on NEW unwaived findings) @echo "=== Phase 847c regression gate (base: $(BASE_REF)) ===" @$(PYTHON) scripts/scan_regression.py --base "$(BASE_REF)" scan-regression-plan: ## Print the surfaces this branch touches as JSON (no scanning; used by CI) @$(PYTHON) scripts/scan_regression.py --base "$(BASE_REF)" --plan # Phase 847f: every waiver's `reachability:` claim names a registered check, and # the check is executed against the live compose files and source tree. A # premise that has stopped being true — a service that was internal-only gains # a published port, a dev dependency gets promoted into a runtime image — fails # here and asks for re-justification. It never deletes the waiver: deleting the # waiver removes the *record* of the exposure, not the exposure. # # `tests/unit/test_reachability_checks.py` runs the same checks under `make # test`; this target is the fast, standalone signal for humans and for the # pre-PR path. reachability-check: tools-image ## Verify every waiver's reachability premise still holds @echo "=== Phase 847f reachability checks ===" @$(TOOLS_RUN) python scripts/reachability_checks.py # Phase 847c: record the baseline the regression gate judges against. Run from # main after images/deps change. First-party images must be built first # ('make build', plus 'make scan-first-party' for the profile-gated ones); a # missing image fails the recording rather than being silently omitted, because # an omitted surface would read as "nothing known here" at gate time. scan-baseline: ## Record security/baseline.json (third-party + first-party images + Python deps) @mkdir -p "$(TRIVY_CACHE)" @echo "=== Recording the CVE baseline ===" @TRIVY_CACHE="$(TRIVY_CACHE)" $(PYTHON) scripts/scan_baseline.py --record --images $(FIRST_PARTY_IMAGES) # Phase 847c: the scheduled monitor (the other half of the split). Full scan, # drift vs the baseline, one deduplicated issue per unwaived finding. Never # blocks a PR — it is deliberately not a required check. Needs the first-party # images built, same as scan-baseline. scan-monitor: ## Full CVE scan, drift report, and (APPLY=1) deduplicated issues @mkdir -p "$(TRIVY_CACHE)" @TRIVY_CACHE="$(TRIVY_CACHE)" $(PYTHON) scripts/scan_baseline_monitor.py --images $(FIRST_PARTY_IMAGES) $(if $(APPLY),--apply,) $(if $(STATE_OUT),--state-out "$(STATE_OUT)",) # Phase 849 (issue #622): GitHub does not dispatch workflows for events caused by # the default GITHUB_TOKEN, so a PR merged by the Actions app lands on main with # no `push` run — the commit that is now the tip was never built. This asks the # one question, and `APPLY=1` repairs a "no" by dispatching ci.yml against main. verify-main: ## Ensure main's tip has a CI run (APPLY=1 dispatches CI when an app-token merge skipped it) $(PYTHON) scripts/ensure_main_verified.py $(if $(APPLY),--apply,) # Phase 847e: measure the MEDIUM vulnerability surface, and record it. The # measurement is what the MEDIUM gating scope was decided from, so it has to be # re-runnable and reproducible: same images, same database, same numbers. # Needs the first-party images built, same as scan-baseline. medium-inventory: ## Measure the MEDIUM surface and record security/baseline-medium.json @mkdir -p "$(TRIVY_CACHE)" @TRIVY_CACHE="$(TRIVY_CACHE)" $(PYTHON) scripts/medium_inventory.py # Detect :latest tags and version drift between compose files. # Runs in < 5 seconds — no external calls, no Docker required. check-image-versions: ## Detect `:latest` tags and version drift across compose files @$(PYTHON) scripts/check_image_versions.py # Lint YAML config and monitoring files with yamllint. # Line-length disabled (long Prometheus expressions are unavoidable). # See .yamllint.yaml for full config. # `monitoring/` moved to `deploy/monitoring/` in phase-205's repo-root cleanup; # this reference was never updated, so on a clean checkout (no repo-root # `monitoring/` dir) `lint-yaml` always crashed with FileNotFoundError. Only # ever passed locally here because of a stray, untracked, root-owned # `monitoring/` directory left over on this host from an unrelated Docker run # — never present in git, so CI always hit this (phase-514). YAML_DIRS := config deploy/monitoring # Containerised in phase-800: this used to call a bare host `yamllint` with no # `command -v` guard — the only linter in the repo not running from a pinned # image. It hard-failed anywhere yamllint was absent, and silently used whatever # version the host happened to have otherwise. Now pinned in Dockerfile.tools. lint-yaml: tools-image ## yamllint `config/` and `monitoring/` @echo "=== yamllint: config and monitoring YAML ===" @$(TOOLS_RUN) yamllint -c .yamllint.yaml $(YAML_DIRS) && echo "✓ YAML lint passed" # Validate Prometheus alert and recording rule files with promtool. # Uses the prometheus container so promtool version matches the deployed version. PROM_RULES := /monitoring/prometheus/alerts.yml \ /monitoring/prometheus/recording_rules.yml \ /monitoring/prometheus/slo_recording_rules.yml \ /monitoring/alertmanager/rules/proxy.rules.yml \ /monitoring/alertmanager/rules/redis.rules.yml \ /monitoring/alertmanager/rules/security.rules.yml \ /monitoring/alertmanager/rules/backup.rules.yml \ /monitoring/alertmanager/rules/ebpf_attack.yml \ /monitoring/alertmanager/rules/management_ui_rules.yml \ /monitoring/alertmanager/rules/slo_alerts.yml \ /monitoring/alertmanager/rules/ti_feed.yml \ /monitoring/alertmanager/rules/tls_alerts.yml \ /monitoring/alertmanager/rules/tap.yml \ /monitoring/alertmanager/rules/performance.rules.yml lint-prom: ## promtool check rules (alerts + recording) @echo "=== promtool check rules: Prometheus alert + recording rules ===" @docker run --rm --entrypoint promtool \ -v "$(PWD)/deploy/monitoring:/monitoring" \ prom/prometheus:v2.48.0 \ check rules $(PROM_RULES) \ && echo "✓ Prometheus rules valid" # Check Python and Go dependencies for known CVEs. # pip-audit scans requirements.txt inside $(TOOLS_IMG); scan-container checks Go. lint-deps: tools-image scan-container ## pip-audit (Python in container) + gosec/govulncheck @echo "=== pip-audit: Python dependency CVEs (containerized) ===" @$(TOOLS_RUN) pip-audit -r requirements.txt || true @$(TOOLS_RUN) pip-audit -r requirements-test.txt || true @$(TOOLS_RUN) pip-audit -r requirements-analytics.txt || true @echo " (CVEs above are advisory — pinned versions may lag; update requirements.txt to remediate)" @echo "" @echo "" @echo "✓ Dependency scan complete" # Scan git history for accidentally committed secrets (gitleaks). # Scans committed files only — working-tree secrets are handled by .gitignore. # False positives are suppressed in .gitleaks.toml with rationale. lint-secrets: ## gitleaks scan of git history @echo "=== gitleaks: scan git history for secrets ===" @docker run --rm \ -v "$(PWD):/repo" \ zricethezav/gitleaks:v8.30.1 detect \ --source /repo --no-banner --config /repo/.gitleaks.toml \ && echo "✓ No secrets found in git history" # Comprehensive Go linting with golangci-lint (errcheck, ineffassign, staticcheck, govet, unused). # Uses the container's own Go — do not pass GOROOT. See .golangci.yaml for config and rationale. # Pinned (was `:latest`, which drifts independently of any code change — a # newer release enabling a new check turned this red with nothing in the diff # to explain it; see phase-514). v1.64.5 (ci.yml's old `go install` pin) can't # run at all here: its own toolchain (go1.24) is older than this repo's target # (go1.26.4) and it refuses to load the config. Pin to v2.11.4 instead — the # same version `:latest` currently resolves to, built with go1.26.1. Bump this # and ci.yml's SAST/CI action pins together when intentionally upgrading. lint-go-full: ## golangci-lint comprehensive @echo "=== golangci-lint: Go code ===" @docker run --rm \ -v "$(PWD):/app" -w /app \ golangci/golangci-lint:v2.12.0 \ golangci-lint run --config .golangci.yaml ./... \ && echo "✓ Go lint passed" # Lint the Redis Lua script with luacheck. # KEYS, ARGV, and redis globals are declared in .luacheckrc (Redis runtime injects them). LUACHECK_SCRIPTS := scripts/sliding_window.lua lint-lua: ## luacheck Redis Lua scripts @echo "=== luacheck: Lua/Redis scripts ===" @docker run --rm \ -v "$(PWD):/repo" \ pipelinecomponents/luacheck \ luacheck --config /repo/.luacheckrc $(addprefix /repo/,$(LUACHECK_SCRIPTS)) \ && echo "✓ Lua lint passed" # Validate JSON files: Grafana dashboards and API spec. # Phase 805: switched the dashboard portion to a glob (was a hand-maintained # list missing 04_capacity.json and ja4proxy-infrastructure.json — neither # had ever been JSON-lint-checked here) so a new dashboard file can't # silently skip this gate again the way tap_sensor.json and this phase's # ci_benchmark_trend.json would have. JSON_FILES := $(wildcard deploy/monitoring/grafana/dashboards/*.json) \ docs/api/openapi.json lint-json: ## JSON syntax validation @echo "=== JSON syntax validation ===" @fail=0; \ for f in $(JSON_FILES); do \ [ -f "$$f" ] || continue; \ printf " %-60s" "$$f"; \ if $(PYTHON) -m json.tool "$$f" > /dev/null 2>&1; then echo "OK"; \ else echo "FAIL"; fail=1; fi; \ done; \ [ $$fail -eq 0 ] || exit 1 @echo "✓ JSON lint passed" # Validate Alertmanager configuration with amtool. # Uses the alertmanager container to match the deployed version. lint-alertmanager: ## amtool check-config @echo "=== amtool check-config: Alertmanager ===" @docker run --rm \ --entrypoint amtool \ -v "$(PWD)/deploy/monitoring:/monitoring" \ prom/alertmanager:v0.26.0 \ check-config /monitoring/alertmanager/alertmanager.yml \ && echo "✓ Alertmanager config valid" # Dummy values for the compose ${VAR:?required} interpolations so `down` can # parse the file even when .env is missing/incomplete (the values are irrelevant # for `down` — it only needs to identify containers by project + service name). # Fixes: `make clean` aborting with "MANAGEMENT_JWT_SECRET is required" in a # freshly-cloned worktree whose .env is incomplete. CLEAN_DUMMY_ENV := MANAGEMENT_JWT_SECRET=_ MANAGEMENT_ADMIN_USER=_ MANAGEMENT_ADMIN_PASSWORD=_ REDIS_PASSWORD=_ GRAFANA_PASSWORD=_ BACKEND_HOST=_ HAPROXY_STATS_USER=_ HAPROXY_STATS_PASSWORD=_ # Clean up (agent-aware: uses .current-agent if set) clean: ## Stop + remove all containers and volumes $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @echo "Cleaning up containers and volumes..." @if [ -n "$(_AGENT)" ]; then \ $(CLEAN_DUMMY_ENV) docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) $$([ -f .env.$(_AGENT) ] && echo --env-file .env.$(_AGENT)) down -v --remove-orphans; \ else \ $(CLEAN_DUMMY_ENV) docker compose -f deploy/docker/docker-compose.poc.yml $$([ -f .env ] && echo --env-file .env) down -v --remove-orphans; \ fi BACKEND_HOST=_ docker compose -f deploy/docker/docker-compose.prod.yml down -v --remove-orphans 2>/dev/null || true # Remove reports and cache directories, ignoring errors (e.g., from container-created files with restrictive permissions) rm -rf reports/ test-results/ __pycache__/ .pytest_cache/ .mypy_cache/ .ruff_cache/ .hypothesis/ .qwen/ 2>/dev/null || true rm -f ja4proxy ja4check ja4proxy-cli bin/ja4p bin/ja4check bin/ja4p-cli 2>/dev/null || true # Full clean rebuild from scratch — wipes volumes, removes built images, rebuilds, starts. # If .current-agent is set (i.e. an agent stack is active), rebuilds and restarts that # agent's stack. Otherwise rebuilds and restarts the default ja4proxy stack. rebuild: clean build ## Wipe volumes/images, rebuild from scratch, start fresh $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @echo "Starting clean rebuild..." @if [ -n "$(_AGENT)" ]; then \ echo " (active agent: $(_AGENT))"; \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) down -v --remove-orphans --rmi local; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env down -v --remove-orphans --rmi local; \ fi docker compose -f deploy/docker/docker-compose.monitoring.yml down -v --remove-orphans --rmi local 2>/dev/null || true @echo "Rebuilding all images (no cache, but preserving apt/pip build caches)..." DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 \ VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env build --no-cache @echo "Starting full stack..." @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) up -d; \ else \ ./scripts/start-all.sh; \ fi # Deploy PoC environment deploy-poc: ## Deploy PoC environment @./scripts/start-poc.sh # Deploy enterprise environment deploy-enterprise: ## Deploy enterprise environment (sudo) @echo "Running enterprise deployment script..." @sudo ./scripts/deploy.sh production # Health checks (agent-aware: uses .current-agent if set) health-check: ## Run health checks against metrics + Redis $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @ENVFILE=$$([ -n "$(_AGENT)" ] && echo ".env.$(_AGENT)" || echo ".env"); \ IP=$$(grep '^AGENT_BIND_IP=' "$$ENVFILE" 2>/dev/null | cut -d= -f2); IP=$${IP:-localhost}; \ RPASS=$$(grep '^REDIS_PASSWORD=' "$$ENVFILE" 2>/dev/null | cut -d= -f2); RPASS=$${RPASS:-changeme}; \ FLAGS=$$([ -n "$(_AGENT)" ] && echo "--project-name ja4_$(_AGENT) --env-file $$ENVFILE" || true); \ echo "Running health checks ($$IP)..."; \ curl -sf "http://$$IP:9090/metrics" > /dev/null && echo "✓ Proxy metrics OK" || echo "✗ Proxy metrics failed"; \ curl -sk "https://$$IP:8443/api/health" > /dev/null && echo "✓ Backend OK" || echo "✗ Backend failed"; \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env $$FLAGS exec -T redis redis-cli -a "$$RPASS" ping > /dev/null 2>&1 && echo "✓ Redis OK" || echo "✗ Redis failed" # View logs (agent-aware: uses .current-agent if set) logs: ## Stream proxy container logs $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) logs -f proxy; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env logs -f proxy; \ fi # Flush all transient security state from Redis (bans, blocks, rate windows, audit logs) # Preserves ja4:whitelist and ja4:blacklist so config survives the flush. # Agent-aware: uses .current-agent if set. flush-redis: ## Reset bans/blocks/rates (keeps whitelist/blacklist) $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @ENVFILE=$$([ -n "$(_AGENT)" ] && echo ".env.$(_AGENT)" || echo ".env"); \ REDIS_PASS=$$(grep '^REDIS_PASSWORD=' "$$ENVFILE" 2>/dev/null | cut -d= -f2); \ if [ -z "$$REDIS_PASS" ]; then echo "✗ No REDIS_PASSWORD in $$ENVFILE"; exit 1; fi; \ FLAGS=$$([ -n "$(_AGENT)" ] && echo "--project-name ja4_$(_AGENT) --env-file $$ENVFILE" || true); \ echo "Flushing Redis security state$$([ -n "$(_AGENT)" ] && echo " (agent: $(_AGENT))")..."; \ COUNT=$$(docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env $$FLAGS exec -T redis redis-cli -a "$$REDIS_PASS" --no-auth-warning \ EVAL "local n=0; \ for _,p in ipairs({'rate:*','banned:*','blocked:*','suspicious:*','enforcement:*','audit:*','repeat_block:*'}) do \ for _,k in ipairs(redis.call('keys',p)) do redis.call('del',k); n=n+1 end \ end; return n" 0 2>/dev/null); \ echo "✓ Cleared $$COUNT keys (whitelist/blacklist preserved)" # ── ja4db feed management ───────────────────────────────────────────────────── # Fetch new malicious fingerprints from FoxIO GitHub / ja4db.com, queue for review fetch-db: ## Fetch new malicious fingerprints from ja4db/FoxIO @./scripts/fetch-ja4db.sh # Show fingerprints awaiting approval list-pending: ## Show fingerprints awaiting admin approval @./scripts/ja4-admin.sh list-pending # Approve all pending fingerprints (prompts for confirmation) approve-all: ## Approve all pending fingerprints @./scripts/ja4-admin.sh approve-all # ── GeoIP monitoring ─────────────────────────────────────────────────────────── # Full blocking report (countries, CIDRs, fingerprints, Prometheus summary) geoip-report: ## Full blocking report @./scripts/ja4-admin.sh report # Run the GeoIP monitor once — auto-blocks attacking countries, respects safe list geoip-monitor: ## Auto-block attacking countries (run once) @./scripts/geoip-monitor.sh # Run geoip-monitor in watch mode (loops every 60s, Ctrl-C to stop) geoip-watch: ## Auto-block attacking countries (continuous loop) @./scripts/geoip-monitor.sh --watch # Download the latest IP2Location LITE country database (run monthly) # Requires proxy restart after update: make stop && make start update-geoip: ## Download latest IP2Location LITE DB (monthly) @./scripts/update-geoip.sh # Check how old the current GeoIP database is (no download) check-geoip: ## Check age of current GeoIP database @./scripts/update-geoip.sh --check # ── Incident response shortcuts (wrappers for scripts/ja4-admin.sh) ────────── # Quick security snapshot: active bans, block totals, top threats attack-status: ## Quick security snapshot @./scripts/ja4-admin.sh status # Top 10 fingerprints by traffic (red = blocked, green = allowed) top-attackers: ## Top 10 fingerprints by traffic @./scripts/ja4-admin.sh top 10 # Blacklist a JA4 fingerprint (instant TCP RST, permanent) # Usage: make block-ja4 FP=t13d190900_9dc949149365_97f8aa674fd9 block-ja4: ## Blacklist a JA4 fingerprint (FP=...) @[ -n "$(FP)" ] || (echo "Usage: make block-ja4 FP="; exit 1) @./scripts/ja4-admin.sh block-ja4 $(FP) # Hard-block an IP for 1 hour # Usage: make block-ip IP=203.0.113.42 block-ip: ## Hard-block an IP address for 1 hour (IP=...) @[ -n "$(IP)" ] || (echo "Usage: make block-ip IP=
"; exit 1) @./scripts/ja4-admin.sh block-ip $(IP) 3600 # Remove all blocks/bans for an IP # Usage: make unblock-ip IP=203.0.113.42 unblock-ip: ## Remove blocks/bans for an IP (IP=...) @[ -n "$(IP)" ] || (echo "Usage: make unblock-ip IP=
"; exit 1) @./scripts/ja4-admin.sh unblock-ip $(IP) # Run performance tests # poc-secrets (phase-800): docker-compose.poc.yml bind-mounts five secret files # that nothing in the repo created. On a machine that had not been hand-primed, # every target raising the PoC stack died with a bare Docker error — # "bind source path does not exist: .../analytics_redis_password.txt" — which is # what aborted `make bench-all` before test-go-perf, load-test and measure-mttr # ever ran. Names are derived from the compose file, so adding a secret there # cannot silently break this again. Existing files are never overwritten. poc-secrets: ## Generate any missing deploy/secrets/*.txt the PoC stack needs @echo "=== PoC secrets ===" @scripts/ensure-poc-secrets.sh perf-test: ## (Retired) Formerly ran Locust performance tests; use make bench-macro @echo "perf-test: Locust performance tests have been retired (Phase 819b)." @echo "Use 'make bench-macro' or 'make bench-hostnative' for truthful proxy load benchmarking." # ── Configuration ───────────────────────────────────────────────────────────── # Set the blocking dial (0 = monitor only, 100 = full blocking) # Usage: make dial LEVEL=50 dial: ## Set blocking dial 0-100 (LEVEL=...) @[ -n "$(LEVEL)" ] || (echo "Usage: make dial LEVEL=<0-100>"; exit 1) @$(PYTHON) scripts/set_dial.py $(LEVEL) # ── Operations ──────────────────────────────────────────────────────────────── # Print SSH tunnel command for the default (non-agent) stack. # For agent stacks use: make tunnel NAME= [HOST=user@server] ssh-tunnels: ## Print SSH tunnel command for default stack @echo "" @echo "NOTE: For agent stacks use: make tunnel NAME= HOST=user@server" @echo "" @echo "Default stack — run this on your laptop:" @echo "" @echo " ssh -L 9091:localhost:9091 -L 9093:localhost:9093 -L 8404:localhost:8404 USER@HOST" @echo "" @echo "Then browse to:" @echo " http://localhost:9091 — Prometheus" @echo " http://localhost:9093 — Alertmanager" @echo " http://localhost:8404/stats — HAProxy stats" @echo " http://localhost:3001 — Grafana" @echo "" # ── Proxy Operations ────────────────────────────────────────────────────────── # Prefer snap Go if present (local dev), else fall back to env/system GOROOT. GOROOT := $(shell if [ -d /snap/go/current ]; then echo /snap/go/current; elif [ -n "$$GOROOT" ]; then echo "$$GOROOT"; else go env GOROOT 2>/dev/null; fi) GO := GOROOT=$(GOROOT) go # Build the Go proxy binary into bin/ja4p # Run all Go unit tests go-test: ## Run all Go unit tests $(GO) test ./... -count=1 # Run all Go tests under the race detector (phase-518). This is the enforcement # arm for the concurrency regression tests — several of them (config hot-reload # snapshot JA4PROXY-2026-0088, forward() config capture 0068, worker lifecycle # 0090) only *detect* a regression when run with -race, and the plain `make test` # gate does not pass -race. Requires cgo (a C toolchain); CI runners have one. test-race: ## Run Go unit tests under the race detector $(GO) test -race ./... -count=1 # Run go vet (static analysis) go-lint: ## Run `go vet` on Go code @if command -v go >/dev/null 2>&1 || [ -x /snap/go/current/bin/go ]; then \ $(GO) vet ./...; \ else \ docker run --rm -v "$$PWD":/src -w /src golang:1.27.3-alpine go vet ./...; \ fi # Audit Python and Go signal scores against registry (Phase 65) check-scores: tools-image ## Audit Python and Go signal scores against registry @$(TOOLS_RUN) python scripts/check-signal-scores.py # ── Security Scans ──────────────────────────────────────────────────────────── # Run OpenSSF Scorecard locally via Docker (Phase 131) scorecard-local: ## OpenSSF Scorecard (local, advisory — the gate is scorecard.yml) @echo "=== OpenSSF Scorecard (Local — advisory; the required gate is the scorecard.yml workflow) ===" @echo "Note: Full checks require a GitHub token with repo scope (make scorecard-local GITHUB_AUTH_TOKEN=...)." @if [ -z "$$GITHUB_AUTH_TOKEN" ]; then \ echo "Warning: GITHUB_AUTH_TOKEN not set — some checks will be skipped or fail."; \ docker run --rm -v $$(pwd):/workspace gcr.io/openssf/scorecard:latest --local /workspace --format default \ || echo " ! scorecard reported issues or could not read the tree (advisory)"; \ else \ docker run --rm -e GITHUB_AUTH_TOKEN=$$GITHUB_AUTH_TOKEN -v $$(pwd):/workspace gcr.io/openssf/scorecard:latest --local /workspace --format default \ || echo " ! scorecard reported issues (advisory)"; \ fi scan-container: ## Run Go SAST (gosec) in a container — gates on high-severity/high-confidence findings @docker build -q -t ja4proxy-security-scan -f deploy/docker/security-scan/Dockerfile . @docker run --rm -v "$(PWD):/app" ja4proxy-security-scan \ "gosec -severity high -confidence high -exclude-dir=.claude -quiet ./..." scan-local: ## Run gosec Go SAST scanner locally @(gosec -fmt=text -exclude-dir=.claude ./... || true) scan: ## Phase 146 — Run all security and container scans @$(MAKE) scan-all @python3 scripts/pipeline_summary.py scan scan-all: scan-container scan-dockerfiles scan-first-party scan-images ## Run all security scans (container, dockerfiles, 1st-party, images) @echo "✓ All blocking scan gates passed (HIGH=0, CRITICAL=0)" # Analyze Docker containers and dependencies for version discrepancies check-updates-container: ## Run dependency update checker in container @echo "=== Containerized Update Check (Go, Python, Docker, Node) ===" @docker build -q -t ja4proxy-update-checker -f deploy/docker/update-checker/Dockerfile . @docker run --rm -v "$(PWD):/app" ja4proxy-update-checker check-updates-local: ## Run dependency update checker locally @echo "=== Local Update Check (Go, Python, Docker, Node) ===" @$(PYTHON) scripts/check_updates.py check-updates: check-updates-container ## Check Python/Go/Docker dependency versions # Start Python legacy proxy alongside the Go proxy for parity comparison. # Both proxies share the same Redis instance. # Go proxy (primary): host port 8081 / metrics 9090 # Python proxy (legacy): host port 8083 / metrics 9094 # Stop Python legacy proxy (Go proxy unaffected) # Confirm Go proxy is HAProxy primary (Go is the default — verify config is correct). # See docs/runbooks/go_proxy_migration.md for background. # Emergency rollback: switch HAProxy to Python legacy proxy. # Run cross-language parity tests. # Both proxies must be running: Python on :8080, Go on :8082. # ── Docker test harness ──────────────────────────────────────────────────────── # Generate ClientHello fixtures (curl + openssl + synthetic) capture-fixtures: ## Generate ClientHello `.bin` fixtures (curl + openssl) @bash scripts/generate_fixtures.sh # Build ja4check utility (reads .bin file, prints JA4 fingerprint) go-build-ja4check: ## Build `bin/ja4check` utility @echo "Building ja4check..." @mkdir -p bin $(GO) build -o bin/ja4check ./cmd/ja4check @echo "✓ bin/ja4check" # Run Go integration tests inside Docker (full test harness, self-contained) test-go-docker: ## Go integration tests inside Docker (self-contained) @echo "Starting Go proxy test stack..." docker compose -f deploy/docker/docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from test-runner test-runner # Run Go integration tests locally (requires Go proxy running on GO_PROXY_PORT) test-go-integration: ## Go integration tests locally $(PYTHON) -m pytest tests/integration/test_go_python_parity.py -v # Run Go chaos tests locally (requires Go proxy running on GO_PROXY_PORT) test-go-chaos: ## Go chaos tests locally $(PYTHON) -m pytest tests/chaos/test_go_proxy_chaos.py -v # Run Go performance benchmarks locally (requires Go proxy running on GO_PROXY_PORT / HOST_PORT_DIRECT) test-go-perf: tools-image ## Go performance benchmarks @docker run --rm --net=host -v $(PWD):/src -w /src $(TOOLS_IMG) pytest tests/performance/test_bench_go_proxy.py -v -s # Run all Go tests: build binaries, generate fixtures, then run test suite test-go: ## Build + test all Go (build binaries, generate fixtures, run suite) $(MAKE) go-build $(MAKE) go-build-ja4check $(MAKE) capture-fixtures $(PYTHON) -m pytest \ tests/integration/test_go_python_parity.py \ tests/chaos/test_go_proxy_chaos.py \ tests/performance/test_bench_go_proxy.py \ -v --tb=short # Phase 201: smoke-test the Go Redis client against a real TLS-terminated Redis. # Generates a self-signed cert, brings up the redis-tls compose overlay, runs the # in-package TLS unit suite (which dials real TLS listeners via the harness), # then tears down. Real TLS handshake; no shortcuts. test-go-redis-tls: ## Go Redis TLS smoke test @bash deploy/docker/redis-tls/generate-certs.sh @docker compose -f deploy/docker/docker-compose.redis-tls.yml up -d --wait @trap 'docker compose -f deploy/docker/docker-compose.redis-tls.yml down -v' EXIT; \ GOROOT=$(GOROOT) go test -race -count=1 ./internal/redis/... -run 'TLS|HealthCheck' test-go-fuzz-smoke: ## Run Go native fuzz smoke tests across all targets $(GO) test -run='^$$' -fuzz='^FuzzClientHello$$' -fuzztime=5s ./cmd/ja4pd/ $(GO) test -run='^$$' -fuzz='^FuzzReadProxyProtocol$$' -fuzztime=5s ./cmd/ja4pd/ $(GO) test -run='^$$' -fuzz='^FuzzReadProxyProtocolV2$$' -fuzztime=5s ./cmd/ja4pd/ $(GO) test -run='^$$' -fuzz='^FuzzFragmentation$$' -fuzztime=5s ./cmd/ja4pd/ $(GO) test -run='^$$' -fuzz='^FuzzProtocolSmuggling$$' -fuzztime=5s ./cmd/ja4pd/ $(GO) test -run='^$$' -fuzz='^FuzzParseClientHello$$' -fuzztime=5s ./internal/tls/ $(GO) test -run='^$$' -fuzz='^FuzzDifferentialClientHello$$' -fuzztime=5s ./internal/tls/ $(GO) test -run='^$$' -fuzz='^FuzzParseClientHelloFeatures$$' -fuzztime=5s ./internal/quic/ $(GO) test -run='^$$' -fuzz='^FuzzReadVarint$$' -fuzztime=5s ./internal/quic/ $(GO) test -run='^$$' -fuzz='^FuzzParseCRYPTOFrames$$' -fuzztime=5s ./internal/quic/ $(GO) test -run='^$$' -fuzz='^FuzzDecodeInitialHeader$$' -fuzztime=5s ./internal/quic/ $(GO) test -run='^$$' -fuzz='^FuzzComputeHMAC$$' -fuzztime=5s ./internal/webhook/ $(GO) test -run='^$$' -fuzz='^FuzzWebhookEndpointRedaction$$' -fuzztime=5s ./internal/webhook/ $(GO) test -run='^$$' -fuzz='^FuzzIsPrivateIP$$' -fuzztime=5s ./internal/webhook/ $(GO) test -run='^$$' -fuzz='^FuzzWebhookEventDelivery$$' -fuzztime=5s ./internal/webhook/ $(GO) test -run='^$$' -fuzz='^FuzzLoadConfigYAML$$' -fuzztime=5s ./internal/config/ $(GO) test -run='^$$' -fuzz='^FuzzLoadYAML$$' -fuzztime=5s ./internal/config/ $(GO) test -run='^$$' -fuzz='^FuzzFlexInt$$' -fuzztime=5s ./internal/config/ $(GO) test -run='^$$' -fuzz='^FuzzExpandEnvVars$$' -fuzztime=5s ./internal/config/ $(GO) test -run='^$$' -fuzz='^FuzzLRUOperations$$' -fuzztime=5s ./internal/cache/ $(GO) test -run='^$$' -fuzz='^FuzzLRUKeyFormatting$$' -fuzztime=5s ./internal/cache/ $(GO) test -run='^$$' -fuzz='^FuzzSNIAnalyzer$$' -fuzztime=5s ./internal/security/ $(GO) test -run='^$$' -fuzz='^FuzzBuildRanger$$' -fuzztime=5s ./internal/security/ $(GO) test -run='^$$' -fuzz='^FuzzEffectiveThresholdAndDecider$$' -fuzztime=5s ./internal/security/ $(GO) test -run='^$$' -fuzz='^FuzzRiskScorer$$' -fuzztime=5s ./internal/security/ $(GO) test -run='^$$' -fuzz='^FuzzPipelineBypassesAndHardBlocks$$' -fuzztime=5s ./internal/security/ $(GO) test -run='^$$' -fuzz='^FuzzVerifyPubSubHMAC$$' -fuzztime=5s ./internal/redis/ $(GO) test -run='^$$' -fuzz='^FuzzPubSubEnvelopeRoundtrip$$' -fuzztime=5s ./internal/redis/ $(GO) test -run='^$$' -fuzz='^FuzzPubSubHandlerDispatch$$' -fuzztime=5s ./internal/redis/ $(GO) test -run='^$$' -fuzz='^FuzzPacketDecoder$$' -fuzztime=5s ./internal/tap/ $(GO) test -run='^$$' -fuzz='^FuzzTCPReassembly$$' -fuzztime=5s ./internal/tap/ $(GO) test -run='^$$' -fuzz='^FuzzParsePortList$$' -fuzztime=5s ./internal/tap/ $(GO) test -run='^$$' -fuzz='^FuzzComputeJA4T$$' -fuzztime=5s ./internal/tap/ $(GO) test -run='^$$' -fuzz='^FuzzCLIArgumentParsing$$' -fuzztime=5s ./internal/cli/engine/ $(GO) test -run='^$$' -fuzz='^FuzzOutputFormatter$$' -fuzztime=5s ./internal/cli/output/ .PHONY: test-go-fuzz-smoke # ── Benchmarking ───────────────────────────────────────────────────────────── bench: ## Run all benchmarks (micro + macro) @$(MAKE) bench-micro @$(MAKE) bench-macro # The trailing `|| true` was removed in phase-800. It had been masking three # permanently-failing benchmarks in ./internal/tls (invalid ClientHello fixtures, # fixed in #408): the package exited 1 on every run and this target still # reported success. `go test` does not run benchmarks without -bench, so nothing # else in the suite could see it either. bench-micro: ## Run Go native micro-benchmarks @echo "=== Go Proxy Micro-benchmarks ===" @GOROOT=$(GOROOT) go test -bench=. -run=^$$ -benchmem ./cmd/ja4pd/ ./internal/tls/ bench-macro: go-build cli-build ## Run end-to-end load test (requires: make start) @echo "=== JA4proxy High-Speed Go Macro-benchmark ===" @PORT=$$(grep "^HOST_PORT_DIRECT=" .env | cut -d= -f2); \ PORT=$${PORT:-8081}; \ echo " Targeting Port: $$PORT"; \ ./bin/ja4p test benchmark --host 127.0.0.1:$$PORT --duration 30 $(ARGS) # Generate SBOM (Software Bill of Materials) for the Go proxy sbom: ## Generate CycloneDX SBOM for Go proxy binary @echo "Generating SBOM for Go proxy..." @if command -v syft &>/dev/null; then \ syft bin/ja4pd -o cyclonedx-json > reports/sbom-proxy.json; \ echo "✓ SBOM generated: reports/sbom-proxy.json"; \ else \ echo "✗ syft not found. Install with: curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh"; \ fi validation-report: ## Generate validation report $(PYTHON) scripts/generate_validation_report.py ## Phase 63 targets — SLO validation and reporting validate-slo-rules: ## Validate SLO recording/alert rules (promtool or YAML) @command -v promtool >/dev/null 2>&1 && { \ promtool check rules deploy/monitoring/prometheus/slo_recording_rules.yml; \ promtool check rules deploy/monitoring/alertmanager/rules/slo_alerts.yml; \ echo "SLO recording rules and alert rules are syntactically valid."; \ } || { \ $(PYTHON) -c "import yaml; yaml.safe_load(open('deploy/monitoring/prometheus/slo_recording_rules.yml')); yaml.safe_load(open('deploy/monitoring/alertmanager/rules/slo_alerts.yml')); print('YAML structurally valid (promtool not installed)')"; \ } slo-report: ## Show SLO report from live Prometheus @echo "=== JA4proxy SLO Report ===" @curl -sg 'http://localhost:9090/api/v1/query?query=job:ja4proxy_availability:ratio_rate5m' \ | $(PYTHON) -c "import sys,json; d=json.load(sys.stdin); v=d['data']['result']; print('Availability (5m):', round(float(v[0]['value'][1])*100, 4) if v else 'NO DATA', '%')" @curl -sg 'http://localhost:9090/api/v1/query?query=job:ja4proxy_latency_p99_good:ratio_rate5m' \ | $(PYTHON) -c "import sys,json; d=json.load(sys.stdin); v=d['data']['result']; print('Latency (5m):', round(float(v[0]['value'][1])*100, 4) if v else 'NO DATA', '%')" @curl -sg 'http://localhost:9090/api/v1/query?query=job:ja4proxy_redis_correctness:ratio_rate5m' \ | $(PYTHON) -c "import sys,json; d=json.load(sys.stdin); v=d['data']['result']; print('Redis (5m):', round(float(v[0]['value'][1])*100, 4) if v else 'NO DATA', '%')" test-slo: ## SLO validation tests GOROOT=$(GOROOT) go test ./internal/metrics/... ./internal/redis/... -count=1 $(PYTHON) -c "import yaml; yaml.safe_load(open('deploy/monitoring/prometheus/slo_recording_rules.yml')); yaml.safe_load(open('deploy/monitoring/alertmanager/rules/slo_alerts.yml')); print('YAML OK')" test-soak: ## High-concurrency connection leak and stream table soak suite (Phase 854) $(GO) test -v -tags=soak -timeout=180s ./cmd/ja4pd ./internal/tap .PHONY: test-soak ## CI local targets ci-local: ## Run the same fast checks the CI workflow runs (Go + Python tests) GOROOT=$(GOROOT) go test -v -coverprofile=coverage.txt -covermode=atomic ./... $(PYTHON) -m pytest tests/ --ignore=tests/integration/test_docker_stack.py -x -q --timeout=60 ## Phase 64a targets smoke-docker: ## Run Docker Compose smoke test bash scripts/smoke/test_docker_compose.sh ## Phase 64b targets smoke-k8s: ## Run Helm + kind smoke test bash scripts/smoke/test_helm_kind.sh ## Phase 64h targets measure-mttr: ## Measure MTTR for DR scenarios bash scripts/measure_mttr.sh ## Phase 86b targets — Load testing harness LOAD_TEST_TARGET ?= localhost:8080 LOAD_TEST_DURATION ?= 60 LOAD_TEST_RPS ?= 1000 LOAD_TEST_SCENARIO ?= mixed load-test: tools-image ## Run JA4proxy load test @docker run --rm --net=host -v $(PWD):/src -w /src $(TOOLS_IMG) python scripts/load_test.py \ --target $(LOAD_TEST_TARGET) \ --duration $(LOAD_TEST_DURATION) \ --rps $(LOAD_TEST_RPS) \ --scenario $(LOAD_TEST_SCENARIO) load-test-baseline: ## Run baseline load test (localhost:8080, 60s, 1000 rps) LOAD_TEST_TARGET=localhost:8080 \ LOAD_TEST_DURATION=60 \ LOAD_TEST_RPS=1000 \ LOAD_TEST_SCENARIO=mixed \ $(MAKE) load-test load-test-report: ## Show latest load test reports @echo "Latest benchmark reports:" @ls -lt test-results/load-test/ 2>/dev/null | head -5 || echo "No reports found" ## Phase 86h targets lint-alert-urls: ## Verify Alertmanager runbook_url values are up to date @echo "Checking alertmanager runbook URLs..." @$(PYTHON) scripts/fix_runbook_urls.py \ --rules-dir deploy/monitoring/alertmanager/rules/ \ --mapping docs/phases/PHASE_86h_runbook_mapping.yml \ --check # Quick-start and performance test targets quick-start: ## Start the proxy with default config (builds if needed) bash scripts/quick-start.sh perf-test-basic: ## Run basic performance test against a local proxy bash scripts/basic_perf_test.sh # Single quality gate — linters + coverage thresholds quality: lint-all lint-coverage ## Run all linters + coverage checks in one shot @echo "" @echo "=== Go coverage check ===" @GOROOT=$(GOROOT) go test -v -coverprofile=coverage.txt -covermode=atomic ./... -coverprofile=/tmp/go_cover_quality.out -count=1 > /dev/null 2>&1 \ && GOROOT=$(GOROOT) go tool cover -func=/tmp/go_cover_quality.out \ | tail -1 | awk '{gsub(/%/,"",$$NF); if($$NF+0 < 80) {print "FAIL: Go coverage "$$NF"% < 80%"; exit 1} else print " ✓ Go coverage "$$NF"%"}' @echo "" @echo "✓ quality complete — all checks passed" verify-finding: ## Run two-state revert proof for a single finding (usage: make verify-finding FINDING=JA4PROXY-2026-0042) @if [ -z "$(FINDING)" ]; then echo "Error: FINDING is required (e.g. make verify-finding FINDING=JA4PROXY-2026-0042)" >&2; exit 1; fi @scripts/verify_revert.sh $(FINDING) verify-findings-all: ## Run two-state revert proof for all findings with recorded regression tests @scripts/verify_revert.sh --all verify-findings: ## Validate docs/security/findings.yaml schema and referential integrity @$(PYTHON) scripts/findings_register.py validate verify-findings-green: ## Run only the regression tests backing findings.yaml entries (fast signal) @$(PYTHON) scripts/findings_register.py verify-regression-tests --stub-green findings-render: ## Regenerate docs/security/FINDINGS_REGISTER.md from findings.yaml @$(PYTHON) scripts/findings_register.py render findings-list: ## List open findings (add FINDINGS_ARGS=... to pass flags, e.g. --severity HIGH) @$(PYTHON) scripts/findings_register.py list $(FINDINGS_ARGS) verify-manifest-closeout: ## Manifest close-out gate — validate register, required docs, ADRs, manifest @$(PYTHON) scripts/phase_121_verify.py # phase-107h: regulatory-conformance regression guards test-compliance-language: ## Phase 107h.1 — fail if "certified"/"compliant" appears in self-assessed compliance docs @$(PYTHON) -m pytest tests/test_compliance_language.py -v test-evidence-paths: ## Phase 107h.2 — fail if conformance docs cite repo paths that don't exist @$(PYTHON) -m pytest tests/test_compliance_evidence_paths.py -v test-compliance: test-compliance-language test-evidence-paths ## Phase 107h — all regulatory-conformance regression guards # phase-107f: ATT&CK mapping CI gate test-attack-mapping: tools-image ## Phase 107f.4 — fail if ATT&CK mapping rows lack confidence labels or cite missing source files @$(TOOLS_RUN) python3 -m pytest tests/test_attack_mapping.py -v # phase-107w.3: doc-link check (requires lychee installed locally — same tool the CI workflow runs) test-doc-links: ## Phase 107w.3 — lychee-check all docs for broken internal links (advisory; gated by docs-link-check.yml) @echo "=== lychee: internal link check (containerised; advisory in make lint) ===" @docker run --rm -v $(PWD):/input -w /input lycheeverse/lychee:0.24.2 \ --no-progress --accept 200,202,204,301,302,403,429 \ --exclude-path /input/archive \ --exclude-path /input/node_modules \ --exclude-path /input/docs/reports/archive \ --exclude-path /input/docs/phases/cancelled \ --exclude-path /input/docs/reports/LINK_AUDIT_2026-06-03.md \ --exclude-path /input/docs/phases/TODO.md \ --exclude-path /input/docs/reference/PROJECT_STATUS.md \ --exclude 'http://localhost' \ --exclude 'http://127\.0\.0\.1' \ "/input/**/*.md" \ || echo " ! lychee reported link issues (advisory — see the docs-link-check workflow for the gate)" lint-meta: tools-image ## Phase 147 — Verify Makefile and automation script health @$(TOOLS_RUN) python3 scripts/meta_lint.py @$(TOOLS_RUN) python3 scripts/check_toolchain_alignment.py @$(TOOLS_RUN) python3 scripts/sync_reference_docs.py --check @$(TOOLS_RUN) python3 scripts/check_deps_resolution.py lint-resolution: tools-image ## Phase 862 — Verify independent Python requirements resolution and parity @$(TOOLS_RUN) python3 scripts/check_deps_resolution.py hygiene: tools-image ## Phase 862 — Fast on-demand local pre-check for CVEs, doc sync, and dependency parity @bash scripts/local_hygiene.sh check-toolchain: tools-image ## Audit toolchain alignment across go.mod, Dockerfiles, and linter capabilities @$(TOOLS_RUN) python3 scripts/check_toolchain_alignment.py reload: ## Reload proxy configuration without restart (SIGHUP) @docker compose -f deploy/docker/docker-compose.poc.yml kill -s SIGHUP proxy 2>/dev/null || kill -s SIGHUP $$(pgrep ja4p) 2>/dev/null || echo "Proxy not running" test-ip: ## Alias for simulating IP decision @./bin/ja4p test ip $(IP) # ── Lint Hierarchy (Phase 92) ───────────────────────────────────────────────── # Prerequisite-based dependency graph — every aggregate lists its leaves as # PREREQUISITES so that `make -n`, dependency parsers, and test suites all see # the correct graph. Advisory targets do not block the gate but report issues. # ── Individual linters (Phase 92 additions) ─────────────────────────────────── lint-pylint: tools-image ## Run pylint errors-only on Python source (advisory) @echo "=== pylint: Python source ===" @$(TOOLS_RUN) python3 -m pylint --errors-only --output-format=colorized \ src/ management/ 2>&1 | tail -30 \ || echo " ! pylint reported errors (advisory — see output above)" @echo " ✓ pylint done" lint-checkov: ## Run Checkov IaC security scan (containerised; advisory) @echo "=== checkov: IaC security scan ===" @docker run --rm -v $(PWD):/tf bridgecrew/checkov:latest \ --quiet --directory /tf --framework dockerfile 2>/dev/null \ || echo " ! checkov reported issues or is unavailable (advisory)" @echo " ✓ checkov done" lint-haproxy: ## Validate HAProxy configuration syntax (advisory) @echo "=== haproxy: config validation ===" @if [ -f deploy/haproxy/haproxy.cfg ]; then \ docker run --rm -v $(PWD):/src haproxy:2.8-alpine \ haproxy -c -f /src/deploy/haproxy/haproxy.cfg 2>&1; \ echo " ✓ HAProxy config valid"; \ else \ echo " ! deploy/haproxy/haproxy.cfg not found — skipping (advisory)"; \ fi lint-helm: ## Run helm lint on Helm charts (advisory) @echo "=== helm lint: Helm charts ===" @if [ -d deploy/helm ]; then \ docker run --rm -v $(PWD):/src alpine/helm:3.17.3 \ lint /src/deploy/helm/ 2>&1 \ || echo " ! helm lint reported issues (advisory)"; \ echo " ✓ helm lint done"; \ else \ true; \ fi lint-markdown: tools-image ## Lint Markdown files for formatting issues (advisory) @echo "=== markdownlint: Markdown files ===" @$(TOOLS_RUN) python3 -m pymarkdown --config /src/.pymarkdown scan docs/ README.md 2>&1 \ || echo " ! markdownlint reported issues (advisory)" @echo " ✓ Markdown lint done" lint-spelling: tools-image ## Spell-check documentation (advisory) @echo "=== codespell: spelling check ===" @$(TOOLS_RUN) codespell docs/ README.md \ --skip="*.po,*.pot,docs/phases,docs/reference,docs/pdf" \ --ignore-words=/src/.codespellignore 2>&1 \ || echo " ! codespell reported issues (advisory)" @echo " ✓ Spelling check done" lint-toml: ## Validate TOML files (pyproject.toml, .gitleaks.toml) using tomllib @echo "=== lint-toml: TOML syntax validation ===" @python3 scripts/lint_toml.py @echo " ✓ TOML lint passed" # checkmake does NOT auto-discover .checkmake.ini from the working directory — # verified 2026-08-10 (Phase 800): with the file in cwd and no flag, maxbodylength # is still reported; passing --config suppresses it. Without --config this repo's # Makefile yields 13 spurious maxbodylength violations; with it, 0. The # docs/phases/complete/PHASE_802.md row claiming cwd auto-discovery was wrong. # # Containerised in phase-800. It previously ran a bare host `checkmake` behind a # `command -v` guard, and nothing in this repo ever installed it — no CI step, no # tools image — so the guard always took the else branch and this step silently # checked nothing, everywhere, for its whole existence. Now it always runs. # checkmake's exit code is the violation count (0 = clean), so no output parsing # is needed; the recipe simply fails on non-zero. lint-makefiles: ## Lint Makefile for common issues (checkmake) @echo "=== Makefile lint ===" @docker run --rm -v "$(PWD):/src:ro" -w /src --entrypoint /checkmake \ $(CHECKMAKE_IMG) --config=.checkmake.ini Makefile @echo " ✓ checkmake passed" lint-go-mod: ## Verify go.mod and go.sum are consistent (go mod verify) @echo "=== go mod verify ===" @if command -v go >/dev/null 2>&1 || [ -x /snap/go/current/bin/go ]; then \ GOROOT=$(GOROOT) $(GO) mod verify; \ else \ docker run --rm -v "$$PWD":/src -w /src golang:1.27.3-alpine go mod verify; \ fi @echo " ✓ go mod verify passed" # ── Aggregate linters ───────────────────────────────────────────────────────── lint-python: lint-static lint-security lint-pylint ## Run all Python linters (ruff, mypy, bandit, pylint) @echo "✓ lint-python complete" lint-go: go-lint lint-go-full lint-go-mod ## Run all Go linters (fmt, vet, golangci-lint, mod verify) @echo "✓ lint-go complete" lint-sast: lint-opengrep lint-checkov ## Run cross-language SAST (Opengrep, Checkov) @echo "✓ lint-sast complete" lint-infra: lint-docker lint-shell lint-yaml lint-lua lint-json lint-haproxy lint-makefiles lint-toml lint-ansible lint-helm ## Run all infrastructure linters @echo "✓ lint-infra complete" lint-observability: lint-prom lint-alertmanager ## Run observability linters (promtool, amtool) @echo "✓ lint-observability complete" lint-supply-chain: lint-secrets lint-deps ## Run supply-chain linters (Gitleaks, govulncheck) @echo "✓ lint-supply-chain complete" doc-health: tools-image ## Validate documentation frontmatter @$(TOOLS_RUN) python3 scripts/check_doc_frontmatter.py lint-docs: doc-health link-check test-attack-mapping ## Core documentation quality checks @echo "✓ lint-docs complete" link-check: ## Alias for the internal-link checker (test-doc-links) @$(MAKE) test-doc-links lint-phases: tools-image ## Validate phase docs (frontmatter, numbering, manifest sync) @$(TOOLS_RUN) python3 scripts/lint-phases.py lint-docs-all: lint-docs lint-phases check-manifest link-check lint-markdown lint-spelling ## Run all documentation quality checks @echo "✓ lint-docs-all complete" lint-opengrep: opengrep-image ## Run Opengrep SAST using local rulesets (containerised) @echo "=== opengrep: SAST (pinned binary v1.33.1 in container) ===" @docker run --rm -u $$(id -u):$$(id -g) -e HOME=/tmp -v $(PWD):/src -w /src $(OPENGREP_IMG) \ scan --quiet --error --disable-version-check --config security/sast/rules/ --config security/sast/custom-rules/ . lint-semgrep: lint-opengrep ## Backward-compatible alias for lint-opengrep (Phase 865) lint-ansible: ## Lint the Ansible playbooks/roles under deploy/ansible (containerised; advisory) @echo "=== ansible-lint: deploy/ansible (advisory — molecule role-path syntax-checks are pre-existing) ===" @docker run --rm -v $(PWD):/src -w /src pipelinecomponents/ansible-lint:0.8.0 \ ansible-lint deploy/ansible/ \ || echo " ! ansible-lint reported issues or is unavailable (advisory)" lint-all: lint-meta lint-python lint-go lint-sast lint-infra lint-observability lint-supply-chain lint-docs-all ## Run every linter in one shot @echo "" @echo "✓ lint-all complete" @python3 scripts/ci_summary.py lint test-lint-hierarchy: tools-image ## Run Phase 92 lint hierarchy structural tests @$(TOOLS_RUN) python3 -m pytest tests/lint-hierarchy/ -v start-poc: deploy-poc ## Alias for starting the POC environment surface-inventory: tools-image ## Rebuild docs/security/ATTACK_SURFACE.md from static AST inspection @$(TOOLS_RUN) python scripts/surface_inventory.py sync: tools-image ## Sync roadmap from manifest.yaml, attack surface, and generated reference docs @$(TOOLS_RUN) python scripts/sync-roadmap.py @$(TOOLS_RUN) python scripts/sync_reference_docs.py @$(TOOLS_RUN) python scripts/surface_inventory.py changelog-assemble: tools-image ## Fold docs/fragments/*.md into CHANGELOG.md (run at release, not per-phase) @$(TOOLS_RUN) python scripts/assemble-changelog.py ja4p-validate: ## Validate proxy configuration YAML @./bin/ja4p config validate @mkdir -p bin @$(GO) build -o bin/ja4p-setup ./cmd/setup setup-build: cli-build ## Build the ja4p CLI the setup wizard needs @true init: setup-build ## Start the guided setup wizard @./bin/ja4p init @echo "✓ bin/ja4ps" bump-build: ## Show build number (derived from git commit count — no file needed) @echo "✓ Build number: $$(git rev-list --count HEAD 2>/dev/null || echo 0)" # Keep in sync: F2 hardened flags — see docs/phases/PHASE_863.md flag matrix go-build: ## Build the Go proxy daemon into bin/ja4pd @mkdir -p bin @command -v gcc >/dev/null 2>&1 || { echo "✗ gcc NOT found — required for hardened build mode (Phase 863)"; exit 1; } VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) ./scripts/go_build_hardened.sh bin/ja4pd ./cmd/ja4pd @echo "✓ bin/ja4pd" # Keep in sync: F2 hardened flags — see docs/phases/PHASE_863.md flag matrix go-build-foss: ## Build royalty-free Go proxy daemon into bin/ja4pd-foss (-tags no_ja4plus) @mkdir -p bin @command -v gcc >/dev/null 2>&1 || { echo "✗ gcc NOT found — required for hardened build mode (Phase 863)"; exit 1; } VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) ./scripts/go_build_hardened.sh bin/ja4pd-foss ./cmd/ja4pd no_ja4plus @echo "✓ bin/ja4pd-foss (pure FOSS, no FoxIO JA4+ methods)" # Keep in sync: F2 hardened flags — see docs/phases/PHASE_863.md flag matrix cli-build: ## Build the unified ja4p CLI tool @mkdir -p bin @command -v gcc >/dev/null 2>&1 || { echo "✗ gcc NOT found — required for hardened build mode (Phase 863)"; exit 1; } VERSION=$(FULL_VERSION) GIT_COMMIT=$(GIT_COMMIT) BUILD_DATE=$(BUILD_DATE) ./scripts/go_build_hardened.sh bin/ja4p ./cmd/ja4p @echo "✓ bin/ja4p" ci-verify: ## CI mirror: full lint, security scans, and test suite gate @$(MAKE) preflight install-hooks: ## Install shared git hooks (pre-push runs `make preflight`) @git config core.hooksPath .githooks @echo "✓ git hooks installed: core.hooksPath=.githooks (pre-push runs 'make preflight')" tunnel: ## Print the SSH local-forward command for an agent stack (NAME=, HOST=) @[ -n "$(NAME)" ] || { echo "Usage: make tunnel NAME= HOST=user@server"; exit 1; } @[ -n "$(HOST)" ] || { echo "Usage: make tunnel NAME= HOST=user@server"; exit 1; } @[ -f ".env.$(NAME)" ] || { echo "No .env.$(NAME) found — run 'make agent-up NAME=$(NAME)' first"; exit 1; } @MGMT=$$(grep '^HOST_PORT_MANAGEMENT=' .env.$(NAME) | cut -d= -f2); MGMT=$${MGMT:-8090}; \ ANL=$$(grep '^HOST_PORT_ANALYTICS=' .env.$(NAME) | cut -d= -f2); ANL=$${ANL:-8080}; \ echo "Run on your laptop to tunnel agent '$(NAME)':"; \ echo " ssh -L $$MGMT:localhost:$$MGMT -L $$ANL:localhost:$$ANL -L 9090:localhost:9090 $(HOST)" # management-* operate on the 'management' service of the current agent stack (.current-agent) management-up: ## Start the management UI for the current agent $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) up -d management; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env up -d management; \ fi management-down: ## Stop the management UI for the current agent $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) stop management; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env stop management; \ fi management-logs: ## Tail the management UI logs for the current agent $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) logs -f management; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env logs -f management; \ fi management-shell: ## Open a shell in the management UI container for the current agent $(eval _AGENT := $(shell cat .current-agent 2>/dev/null)) @if [ -n "$(_AGENT)" ]; then \ docker compose -f deploy/docker/docker-compose.poc.yml --project-name ja4_$(_AGENT) --env-file .env.$(_AGENT) exec management sh; \ else \ docker compose -f deploy/docker/docker-compose.poc.yml --env-file .env exec management sh; \ fi test-ratio: ## Show the test-to-code ratio @$(PYTHON) scripts/test_ratio.py test-component-suites: ## Run every component test suite (unit, chaos, adversarial) @$(MAKE) test-unit test-chaos test-adversarial bench-all: ## Run every heavy benchmark (load, go-perf, hostnative) — slow, runs alone @echo "=== bench-all: heavy benchmarks (excluded from 'make lint scan test') ===" @$(MAKE) bench @$(MAKE) test-go-perf @$(MAKE) load-test @$(MAKE) bench-hostnative @echo "✓ bench-all complete" verify-all: ## Full release gate: lint + scan + test + bench-all — slow @$(MAKE) lint @$(MAKE) scan @$(MAKE) test @$(MAKE) bench-all @echo "✓ verify-all complete" bench-hostnative: go-build cli-build ## End-to-end throughput, ja4pd host-native (no docker-proxy; ~4.5x the bridge port) @chmod +x scripts/bench-hostnative.sh @./scripts/bench-hostnative.sh scan-js: ## Scan the vendored Management-UI JS for known CVEs (retire.js) @echo "-> retire.js CVE scan of management/static (vendored front-end JS)" @npx --yes retire@5 --path management/static lane: ## Show this worktree's dev lane (collision-free host ports + Grafana URL) @chmod +x scripts/lane-env.sh && ./scripts/lane-env.sh open: lane ## Open a lane service: make open SVC=grafana|management|metrics|prometheus @set -a; . ./.env; set +a; \ svc="$(or $(SVC),grafana)"; bind="$${AGENT_BIND_IP:-127.0.0.1}"; \ case "$$svc" in \ grafana) port=$$HOST_PORT_GRAFANA;; management) port=$$HOST_PORT_MANAGEMENT;; \ metrics) port=$$HOST_PORT_METRICS;; prometheus) port=$$HOST_PORT_PROMETHEUS;; \ *) echo "unknown SVC=$$svc (grafana|management|metrics|prometheus)"; exit 1;; esac; \ url="http://$$bind:$$port"; echo "opening $$svc -> $$url"; \ (xdg-open "$$url" >/dev/null 2>&1 || open "$$url" >/dev/null 2>&1 || echo " open it manually: $$url") loadtest: go-build cli-build ## Lane-isolated good/bad load test -> watch Grafana (knobs: GOOD_RATE BAD_RATE DURATION WORKERS DIAL) @chmod +x scripts/lane-env.sh && ./scripts/lane-env.sh >/dev/null @$(MAKE) --no-print-directory start-poc @set -a; . ./.env; set +a; \ g="$(or $(GOOD_RATE),10)"; b="$(or $(BAD_RATE),190)"; d="$(or $(DURATION),300)"; \ w="$(or $(WORKERS),16)"; dial="$(or $(DIAL),100)"; bind="$${AGENT_BIND_IP:-127.0.0.1}"; \ echo "lane $$JA4_LANE: raising dial to $$dial (best-effort via Management API) ..."; \ JA4PROXY_URL="http://$$bind:$$HOST_PORT_MANAGEMENT" JA4PROXY_TOKEN="$${JA4PROXY_TOKEN:-$$UI_API_KEY}" \ ./bin/ja4p management dial set $$dial --confirm 2>/dev/null \ || echo " (could not auto-set dial; scored/risk panels still populate. Set it via 'ja4p management dial' for block/ban panels)"; \ echo "driving good=$$g/s bad=$$b/s for $${d}s at 127.0.0.1:$$HOST_PORT_DIRECT ..."; \ ./bin/ja4p test benchmark --host "127.0.0.1:$$HOST_PORT_DIRECT" --good-rate $$g --bad-rate $$b --workers $$w --duration $$d; \ echo "watch it: http://$$bind:$$HOST_PORT_GRAFANA (Grafana)" # ── Phase 332: pre-PR gate ─────────────────────────────────────────────────── # Run the full required-check set locally before opening a PR. This is the # shift-left half of the CI trial: heavy checks (lint, scan) run here on the # author's machine so the PR path can stay fast. All three must be green before # you push a branch and open a PR. See docs/phases/complete/PHASE_332.md. check: go-build cli-build compose-validate test ## Fast gate: compile + env validate + tests (~3 min, no image builds or CVE scans) @echo "✓ check passed — matches CI required checks" preflight: ## Full local gate before opening a PR: lint + scan + test + test-race (~25 min) @echo "NOTE: scan-first-party scans Docker images — run 'make build' first to ensure they are current." @echo "=== preflight [1/4] make lint ===" @$(MAKE) lint @echo "=== preflight [2/4] make scan ===" @$(MAKE) scan @echo "=== preflight [3/4] make test ===" @$(MAKE) test @echo "=== preflight [4/4] make test-race ===" @$(MAKE) test-race @echo "✓ preflight passed — safe to open a PR" tap-build: ## Build standalone TAP sensor binary GOBIN=$(GOPATH)/bin go build -o bin/ja4-tap ./cmd/ja4-tap # ── Phase 245: First-class admin CLI + minimal init ────────────────────────── admin: ## Run the ja4-admin incident response CLI (pass ARGS for commands) @./scripts/ja4-admin.sh $(ARGS) init-minimal: setup-build ## Emergency init: prompt for BACKEND_HOST, generate .env, done @./bin/ja4p init --minimal # ── Phase 511: Emergency traffic insertion / rollback ──────────────────────── # Routes incoming :443 through JA4proxy :8443 using iptables PREROUTING. # Requires sudo. Rollback is instant — see docs/runbooks/dashboard_access.md. TRAFFIC_PUBLIC_PORT ?= 443 TRAFFIC_PROXY_PORT ?= 8443 traffic-on: ## Insert JA4proxy into the traffic path (iptables redirect :443→:8443; needs sudo) @echo "=== Routing port $(TRAFFIC_PUBLIC_PORT) → JA4proxy :$(TRAFFIC_PROXY_PORT) ===" @sudo iptables -t nat -I PREROUTING -p tcp --dport $(TRAFFIC_PUBLIC_PORT) -j REDIRECT --to-port $(TRAFFIC_PROXY_PORT) @echo "✓ Live traffic now flowing through JA4proxy." @echo " Monitor: docker compose logs -f ja4proxy" @echo " Rollback at any time: make traffic-off" traffic-off: ## Remove JA4proxy from the traffic path (instant rollback; needs sudo) @echo "=== Removing JA4proxy from traffic path ===" @sudo iptables -t nat -D PREROUTING -p tcp --dport $(TRAFFIC_PUBLIC_PORT) -j REDIRECT --to-port $(TRAFFIC_PROXY_PORT) 2>/dev/null \ && echo "✓ Traffic bypassing JA4proxy. Original service restored." \ || echo "Rule not found — JA4proxy was not in the traffic path." # ── Phase 92: PHONY declarations for lint hierarchy targets ────────────────── .PHONY: lint-pylint lint-checkov lint-haproxy lint-helm lint-markdown lint-spelling .PHONY: lint-toml lint-makefiles lint-go-mod test-lint-hierarchy .PHONY: lint-python lint-go lint-sast lint-infra lint-observability .PHONY: lint-supply-chain lint-docs-all lint-all lint-docs doc-health link-check # ── Phase 814a: penetration-testing range ──────────────────────────────────── pentest-range: ## Bring up the isolated pentest range (zero egress, verified) and print provenance @./scripts/start-pentest-range.sh pentest-range-down: ## Tear down the pentest range (evidence is kept) @./scripts/start-pentest-range.sh --down pentest-range-verify: ## Re-run the range isolation assertions without rebuilding @./scripts/start-pentest-range.sh --verify-only pentest-shell: ## Open a shell on the attacker workstation inside the range @docker compose -p ja4range --env-file .pentest-range/range.env \ -f deploy/docker/docker-compose.poc.yml \ -f deploy/docker/docker-compose.pentest.yml exec attacker bash .PHONY: pentest-range pentest-range-down pentest-range-verify pentest-shell # Phase 824 — journey tests. Assert a PERSON can do the thing, not that a unit # returns a value. Require a running stack, so they are not part of `make test`. # TIER: 1=static/contract only, 2=live stack (default), 3=fault injection / restart TIER ?= 2 test-journeys: ## Phase 824 — run customer-journey checks against a live stack (TIER=2 default) @set -a; [ -f .env ] && . ./.env; set +a; \ fail=0; \ req_tier="$(TIER)"; \ req_num="$${req_tier#T}"; req_num="$${req_num#t}"; \ echo "Running customer journey checks for TIER <= $$req_num..."; \ for j in tests/integration/journeys/check_*.sh; do \ [ -x "$$j" ] || continue; \ script_tier=$$(grep -E '^# Tier: T[0-9]+' "$$j" | head -1 | sed -E 's/.*T([0-9]+).*/\1/' || echo "2"); \ [ -n "$$script_tier" ] || script_tier=2; \ if [ "$$script_tier" -le "$$req_num" ]; then \ echo "── $$j (Tier: T$$script_tier)"; "$$j" || fail=1; \ fi; \ done; \ [ $$fail -eq 0 ] || { echo "✗ journey check(s) failed"; exit 1; }; \ echo "✓ all journey checks for TIER=$$req_num passed" test-ui-contract: tools-image ## Phase 824f — run Tier 1 UI route contract, regression, and health tests @echo "=== Running Tier 1 UI tests (containerized: $(TOOLS_IMG)) ===" @$(TOOLS_RUN) pytest \ management/tests/test_partial_contract.py \ management/tests/test_ui_regression.py \ management/tests/test_ui_health.py \ -v --timeout=60 --tb=short # ── phase-826: demo support ─────────────────────────────────────────────────── # Pre-flight before showing the product to anyone. Checks the failures that # have actually happened and were invisible: analytics unable to read the # stream, events rejected on an HMAC mismatch, the proxy never loading its # blacklist, an empty Intelligence panel. See docs/DEMO_RUNBOOK.md. .PHONY: demo-check demo-bot demo-check: ## Pre-flight health check for demo environment @scripts/demo-check.sh # One deliberately non-browser TLS connection, for the "now something that # isn't a browser" step of the demo. demo-bot: ## Send test non-browser TLS connection for demo @scripts/demo-bot.sh # ── Phase 816: End-to-end management console showcase demo ────────────────── .PHONY: demo demo-verify demo-stop demo: poc-secrets ## Phase 816 — Launch showcase demo environment with API seeding and traffic @bash scripts/demo-mgmt.sh demo-verify: ## Phase 816 — Run end-to-end data flow and policy assertions on demo stack @bash scripts/demo-verify.sh demo-stop: ## Phase 816 — Gracefully stop all demo environment services @bash scripts/stop-all.sh