# ═══════════════════════════════════════════════════════════════════ # ✦ T L A M A T I N I ✦ — "one who knows" # # Created by Angela López Mendoza · @angelahack1 # Developer · Architect · Creator of Tlamatini # # Every line of this file was written by Angela López Mendoza. # ═══════════════════════════════════════════════════════════════════ # Tlamatini Author Banner — do not remove (releases scrub the name automatically) """ copy_source_assets.py — generate the TlamatiniSourceCode self-modify snapshot. Called by `build.py --self-modify` (and runnable standalone) to assemble the COMPLETE source-code tree Tlamatini needs to take -> modify -> integrate -> regenerate her own Tlamatini.exe, while omitting heavy media and everything already present (or regenerable) in the installed directory tree. DEEP ASSET ANALYSIS (what a self-rebuild actually needs) ======================================================== The snapshot mirrors the repo-root layout, so a rebuild is simply: cd /TlamatiniSourceCode && python build.py --self-modify INCLUDED — everything the build pipeline and the app's source surface touch: * Build pipeline (repo root): build.py, build_installer.py, build_uninstaller.py, versioning.py, install.py, uninstall.py, regen_secrets.py, requirements.txt, pyinstaller_hooks/, eslint.config.mjs, package.json, pnpm-lock.yaml, tlamatini_mcp_server.py, this script itself. * Self-update capability ("About > Check for updates"): the in-app updater agent/self_update.py AND the external file-swap script apply_update.ps1 (repo root; build.py ships it next to Tlamatini.exe so self_update.py can copy it out and run the swap). Both are plain source and ship verbatim. * ALL PowerShell helpers (*.ps1): Tlamatini.ps1, CreateShortcut.ps1, RemoveShortcut.ps1, register_flw.ps1, unregister_flw.ps1, apply_update.ps1 (+ CreateShortcut.json). * The full Django tree: every .py, .html, .css, .js, .yaml/.yml, .json, .pmt (prompt.pmt + monitoring-prompt.pmt), .proto, SKILL.md packages, all installed agent templates, migrations, tests, AuxTests/. * Firmware template sources (.ino/.h/.cpp — ArduinoTemplateProject etc.). * Docs that drive behavior or onboarding: CLAUDE.md, docs/claude/*, agents_descriptions.md (parsed at runtime for tooltips!), README.md, BookOfTlamatini.md, ACPX.md, VERSIONING.md, .agents/, .mcps/, .skills/, .claude/ (skills + hooks; settings.local.json EXCLUDED — secret fragments), .github/, LICENSE, .gitignore, .mcp.json. * Small build-REQUIRED binaries: *.ico (build.py --icon + console icon, ~39 KB x2), *.wav (notification/hypervisor sounds shipped via --add-data static, ~730 KB total), *.svg (spinner). A rebuild FAILS or produces a degraded app without these, so they ship despite being binary. EXCLUDED — heavy media, secrets, and regenerable/duplicated state: * Media: .pdf, .pptx, .png, .jpg/.jpeg, .gif, .mp4 ... (~44 MB — the agent/images gallery, demo videos, presentation, summary PDF). None are inputs to PyInstaller except agent/static/agent/video/XAIHT-Tlamatini.mp4, which is listed in the RESTORE manifest (copy back from the installed _internal/agent/static tree before rebuilding). * jd-cli/jd-cli.jar (3.8 MB) — already at /jd-cli verbatim; listed in the RESTORE manifest (build.py hard-requires it). * Tlamatini/staticfiles/ — regenerated by `manage.py collectstatic` (rebuild instruction), never a source of truth. * Secrets & local state: data.keys, db.sqlite3, .claude/settings.local.json, key.txt/cookie.txt/crumb.json, *.session, *.log, *.pos, _version.py, *.version.txt, .build.lock, pkg.zip/*.zip, *.exe/*.dll/*.pyd, *.lnk. * Dev/derived trees: .git, __pycache__, node_modules, venv/.venv, build/, dist/, agents/pools/, Temp/, Templates/, content_generated/, context_files/, applications/, caches — and TlamatiniSourceCode itself (recursion guard). * SECRET REDACTION: agent/config.json and the agent config.yaml files may carry LIVE keys on a dev machine. Values for secret-looking keys are replaced with the regen_secrets.py placeholder style ""; at rebuild time Tlamatini restores them from the installed config.json (see _REBUILD_INSTRUCTIONS.md written into the snapshot). The snapshot also gets `_SOURCE_SNAPSHOT_MANIFEST.json` (counts, sizes, restore list, redacted-file list) and `_REBUILD_INSTRUCTIONS.md` (the exact restore + rebuild runbook Tlamatini follows). Usage: python copy_source_assets.py # -> Tlamatini/agent/TlamatiniSourceCode python copy_source_assets.py --dest # -> custom destination python copy_source_assets.py --no-redact # keep secrets verbatim (LOCAL ONLY) From build.py: import copy_source_assets stats = copy_source_assets.copy_source_assets(repo_root, dest_dir) """ from __future__ import annotations import argparse import fnmatch import json import os import re import shutil import sys import time from datetime import datetime from pathlib import Path from typing import Any, Dict, List REPO_ROOT = Path(__file__).resolve().parent DEFAULT_DEST = REPO_ROOT / "Tlamatini" / "agent" / "TlamatiniSourceCode" # ── Exclusion rules ────────────────────────────────────────────────────────── # Directory NAMES skipped wherever they appear in the tree. EXCLUDED_DIR_NAMES = { ".git", "__pycache__", "node_modules", ".venv", "venv", "env", ".ruff_cache", ".pytest_cache", ".mypy_cache", ".pyre", ".pytype", "htmlcov", ".idea", ".vscode", ".eggs", "build", "dist", # PyInstaller outputs "pools", ".tlamatini", # agent runtime pools/metadata (state, not source) "staticfiles", # regenerated by collectstatic "Temp", "Templates", # runtime scratch / scaffolded projects "security_logs", # security defender alert logs + asset-test run artifacts (runtime state) "content_generated", "context_files", "applications", "agents_backup", # self-update one-generation backup of agents/ "updater", # self-update staging/run dir (apply_update.ps1) "TlamatiniSourceCode", # recursion guard — never snapshot a snapshot "python", # /python: source-tree build Python (carried-source artifact, gitignored) "Go", "gopath", "go-build", # Discoverer's private Go toolchain + caches (gitignored **/Go/; ~150 MB of READ-ONLY module-cache files — never snapshot) } # Exact source directories whose names otherwise match generated-output rules. # PDF.js's build/ contains the shipped API and worker, not our build output. KEEP_DIR_PATHS = { "Tlamatini/agent/static/agent/vendor/pdfjs/build", "Tlamatini/agent/static/agent/vendor/frontend/bootstrap/dist", "Tlamatini/agent/static/agent/vendor/frontend/jquery/dist", "Tlamatini/agent/static/agent/vendor/frontend/dompurify/dist", } # File EXTENSIONS skipped everywhere (lowercase, with dot). EXCLUDED_EXTENSIONS = { # documents / presentations ".pdf", ".pptx", ".ppt", # images (NOTE: .ico and .svg are deliberately KEPT — build-required) ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tif", ".tiff", # audio/video (NOTE: .wav is deliberately KEPT — shipped UI sounds) ".mp4", ".avi", ".mov", ".mkv", ".webm", ".mp3", ".ogg", ".flac", # archives / binaries ".zip", ".7z", ".gz", ".rar", ".jar", ".exe", ".dll", ".pyd", ".so", ".lnk", ".msi", # PARTIAL / IN-PROGRESS WRITES. A FAILED build leaves `pkg.zip.part` in the # repo ROOT, where no directory rule reaches it, and `.zip` above does not # match `.part`. On 2026-09-19 that let a 1 GiB fragment of the previous # zip into the snapshot, so pkg.zip came out at 2.93 GB against a 1.99 GB # budget - and because THAT build also failed it left an even bigger # `.part` behind for the next one. A build artifact is never source. ".part", ".partial", ".tmp", ".crdownload", # generated / local state ".sqlite3", ".log", ".session", ".pos", ".lock", ".pyc", ".pyo", ".spec", ".manifest", ".mo", ".pot", } # Exact file NAMES skipped wherever they appear. EXCLUDED_FILE_NAMES = { ".torrenter-receipt", "torrenter-result.json", "db.sqlite3", "data.keys", "key.txt", "cookie.txt", "crumb.json", "package-lock.json", "settings.local.json", "_version.py", ".build.lock", "Thumbs.db", ".DS_Store", # PII / private local state (gitignored) — NEVER snapshot. contacts.private.json # holds Angela's REAL phone numbers (the private build bundles it as the install's # contacts.json via TLAMATINI_BUNDLE_CONTACTS — it is user data, not source); # .private_targets.json is her leak-target list (names/phones/emails); a live # contacts.json is user state a rebuild never needs (build.py regenerates it). "contacts.private.json", "contacts.json", # The private release's External-MCP catalog: the servers Angela really uses, # with their live tokens (see build_complete_private_release.py). "external_mcps.private.json", ".private_targets.json", "private_targets.json", # Leak-audit artifacts (gitignored) — carry found-secret SAMPLES by design, # so shipping them into the snapshot re-leaks what they flagged. "private_data_audit_report.json", "personal_info_audit_report.md", "public_release_verify_report.json", "_tlamatini_agents_manifest.json", # generated installed-path/version metadata } # Glob patterns (matched against the file name) skipped everywhere. EXCLUDED_FILE_GLOBS = ("*.version.txt", "*_log.txt", "*_log.log", "pkg.zip*") # ── Credential files — HARD DROP, never resurrectable ──────────────────────── # This walk reads the WORKING TREE, not git. So a .gitignore'd secret file is # still physically on disk and WAS being copied into TlamatiniSourceCode/ — and # from there into pkg.zip, and out to every user of a `--self-modify` build. # Git history stays clean while the BUILD leaks: exactly the hole found live on # 2026-07-31, when the snapshot was carrying `open_router.key` (a real OpenRouter # sk-or-v1 key) and `.creds.env` (the chat-test TLAMATINI_USER/PASS login). # A rebuild NEVER needs a credential file — build.py reads none of these. # Enforced in the SAME tier as EXCLUDED_FILE_NAMES (i.e. BEFORE KEEP_PATH_GLOBS), # so a KEEP carve-out can never resurrect a secret. SECRET_FILE_EXTENSIONS = { ".key", ".keys", ".pem", ".p12", ".pfx", ".jks", ".keystore", ".env", # covers "*.env" (e.g. .creds.env) ".asc", ".gpg", ".ppk", } # Name globs for credential files an extension test cannot see: `Path(".env").suffix` # is "" (a leading dot makes the whole name a STEM), so a bare `.env` needs a name # rule. Kept deliberately NARROW — a tempting `*_secrets.*` would silently drop # `regen_secrets.py`, a REQUIRED build script. SECRET_FILE_GLOBS = ( ".env", ".env.*", "id_rsa", "id_rsa.*", "id_ed25519", "id_ed25519.*", ) # ── DROP carve-outs (path-based) ───────────────────────────────────────────── # Snapshot-relative POSIX path globs ALWAYS dropped: run artifacts that are # neither source nor a build input, and that a rebuild never reads. Kept narrow # and PATH-anchored on purpose — a bare "reports" entry in EXCLUDED_DIR_NAMES # would be far too broad and could drop real source elsewhere in the tree. DROP_PATH_GLOBS = ( "artifacts/**", # local generated document/style previews, not rebuild source # dated chat-test run artifacts (gitignored; ~1.2 MB of HTML/JSONL per sweep) ".claude/skills/tlamatini-daily-chat-test/harness/reports/**", ) # ── KEEP carve-outs (path-based) ───────────────────────────────────────────── # Snapshot-relative POSIX path globs whose files are KEPT even when # EXCLUDED_EXTENSIONS / EXCLUDED_FILE_GLOBS would drop them. Same rationale as # the global .ico / .wav / .svg keeps: these are SMALL, SHIPPED assets the app # and its shipped docs reference BY NAME, so a rebuild without them produces a # visibly broken Tlamatini — they are source, not media. # * static/agent/img/** -> the chat-avatar frames referenced from # agent_page.html via {% static %} (~152 KB). # * agents/*/guide_assets/** -> the setup screenshots embedded in the shipped # HOW_TO_GET_YOUR_*_ASSETS.md guides (~1.2 MB). # Deliberately NOT covered: static/agent/video/** (large, restore-mapped) and # agent/images/** (the ~70 MB gallery — optional, degrades gracefully). # PDF.js vendor assets also ship intact, including its decoder/font binaries. # KEEP_DIR_PATHS separately permits its otherwise-excluded build/ directory. # EXCLUDED_FILE_NAMES (secrets / PII / local state) is NEVER overridden here. KEEP_PATH_GLOBS = ( "Tlamatini/agent/static/agent/img/**", "Tlamatini/agent/agents/*/guide_assets/**", "Tlamatini/agent/agents/pdfer/_art/**", # tracked PDFer rendering ornaments, not gallery art "Tlamatini/agent/static/agent/vendor/pdfjs/**", "Tlamatini/agent/static/agent/vendor/frontend/**", ) # ── Completeness guarantee ─────────────────────────────────────────────────── # The walk is generic (any new .py/.yaml/.js/... ships automatically), but a # careless future exclusion rule could silently drop a build- or run-critical # source. These snapshot-relative paths MUST land in every snapshot; a missing # one raises rather than producing a quietly-incomplete self-modify tree. # Add the canonical file of any new always-present capability here. REQUIRED_SNAPSHOT_FILES = ( "Tlamatini/agent/canvas_file_views.py", "Tlamatini/agent/native_dialogs.py", "Tlamatini/agent/test_canvas_file_views.py", # Preserve the source-quality commands for builds initiated from the snapshot. "scripts/check_python_quality.py", "scripts/check_js_parse.mjs", "scripts/file_drop_router_cases.mjs", "scripts/menu_reference_graph.mjs", "Tests/test_avatar_animation.cjs", "Tests/test_avatar_visible.cjs", "Tests/test_release_regeneration.py", "Tests/test_setup_release_paths.py", "package.json", "eslint.config.mjs", # Main-chat Views: preserve the complete feature and its visible check. "Tlamatini/agent/templates/agent/agent_page.html", "Tlamatini/agent/static/agent/js/agent_page_layout.js", "Tlamatini/agent/static/agent/js/avatar_size.js", "Tlamatini/agent/static/agent/js/agent_page_canvas.js", "Tlamatini/agent/static/agent/js/agent_page_init.js", "scripts/chat_views_visible.py", "scripts/chat_views_frozen_visible.py", 'Tlamatini/agent/templates/admin/base_site.html', 'Tlamatini/agent/static/agent/css/admin_theme.css', 'Tlamatini/agent/static/agent/css/ui_tokens.css', 'Tlamatini/agent/static/agent/css/ui_shell.css', 'Tlamatini/agent/static/agent/css/ui_splitters.css', 'Tlamatini/agent/static/agent/css/entry_pages.css', 'Tlamatini/agent/static/agent/js/ui_splitters.js', # Release lifecycle repairs and their real source/frozen acceptance harness. "uninstall.py", "uninstall_processes.py", "Tlamatini/agent/test_uninstaller_lifecycle.py", "Tlamatini/agent/test_uninstall_processes.py", "Tlamatini/agent/test_mcp_configured_endpoints.py", ".codex/skills/tlamatini-release-validation/SKILL.md", ".codex/skills/tlamatini-release-validation/references/acceptance-matrix.md", "scripts/installed_panels_visible.py", "scripts/release_auth_checks.py", "scripts/release_shell_file_checks.py", "scripts/release_extended_checks.py", "scripts/release_agent_catalog_checks.py", "scripts/release_acp_editor_checks.py", "scripts/release_acp_runtime_checks.py", "scripts/release_commentary_checks.py", "scripts/release_output_resize_checks.py", "scripts/release_prompt_runtime_checks.py", "scripts/release_local_agent_checks.py", "scripts/release_mcp_checks.py", "Tlamatini/agent/flow_file_open.py", "Tlamatini/agent/flow_file_views.py", "Tlamatini/agent/test_flow_file_open.py", "Tlamatini/agent/test_flow_file_views.py", "flow_file_associations.ps1", "register_fpmt.ps1", "unregister_fpmt.ps1", "register_flw.ps1", "unregister_flw.ps1", "docs/windows-flow-files.md", "scripts/flow_file_associations_test.ps1", "scripts/flow_files_visible.py", # Prompt Flow Panel: execution, UI, regression coverage and portable example. "Tlamatini/agent/prompt_flow_panel_consumer.py", "Tlamatini/agent/prompt_flow_panel_runtime.py", "Tlamatini/agent/services/prompt_flow_panel.py", "Tlamatini/agent/templates/agent/prompt_flow_panel.html", "Tlamatini/agent/static/agent/css/prompt_flow_panel.css", "Tlamatini/agent/static/agent/css/flow_canvas.css", "Tlamatini/agent/static/agent/js/flow-canvas-interactions.js", "Tlamatini/agent/static/agent/js/prompt-flow-panel-model.js", "Tlamatini/agent/static/agent/js/prompt-flow-panel.js", "Tlamatini/agent/test_prompt_flow_panel.py", "Tlamatini/agent/test_prompt_flow_panel_runtime.py", "Tlamatini/agent/test_prompt_flow_panel_carriage.py", "Tlamatini/agent/test_prompt_flow_panel_websocket.py", "scripts/menu_reference_graph.mjs", "scripts/menu_browser_checks.py", "scripts/menu_live_checks.py", "scripts/panel_search_title_visible.py", "scripts/acp_editor_visible.py", "scripts/prompt_flow_extension_visible.py", "scripts/prompt_flow_selection_visible.py", "scripts/prompt_flow_connections_visible.py", "scripts/prompt_flow_commentary_visible.py", "scripts/flow_canvas_mechanics_visible.py", "Tlamatini/agent/static/agent/js/acp-editor-tools.js", "scripts/run_menu_state_checks.py", "Tlamatini/agent/management/commands/check_prompt_flow_panel.py", "Tlamatini/agent/management/commands/check_terminater.py", "docs/torrenter.md", "Tlamatini/agent/management/commands/enroll_torrenter.py", "Tlamatini/agent/migrations/0221_torrenter.py", "Tlamatini/agent/migrations/0222_replace_mistral_large3.py", "Tlamatini/agent/rag/directory_listing.py", "Tlamatini/agent/user_creation_wizard.py", "Tlamatini/agent/test_user_creation_consumer.py", "Tlamatini/agent/test_user_creation_command.py", "Tlamatini/agent/test_user_creation_wizard.py", "Tlamatini/agent/management/commands/createuserconsole.py", "Tlamatini/agent/migrations/0223_create_superuser_console_handoff.py", "Tlamatini/agent/test_available_directories.py", "Tlamatini/agent/torrenter_tool.py", "Tlamatini/agent/torrenter_bridge.py", "Tlamatini/agent/torrenter_views.py", "Tlamatini/agent/templates/agent/torrenter_sources.html", "Tlamatini/agent/agents/torrenter/config.yaml", "Tlamatini/agent/agents/torrenter/torrenter.py", "Tlamatini/agent/agents/torrenter/client.py", "Tlamatini/agent/agents/torrenter/contract.py", "Tlamatini/agent/agents/torrenter/sources.py", "Tlamatini/agent/agents/torrenter/activity.py", "Tlamatini/agent/test_torrenter.py", "Tlamatini/agent/test_torrenter_delivery.py", "Tlamatini/tests_e2e/torrenter_console_windows.py", "Tlamatini/tests_e2e/torrenter_books.py", "Tlamatini/tests_e2e/torrenter_live.py", "Tlamatini/tests_e2e/torrenter_panel.py", "Tlamatini/agent/test_executer_window_report.py", "Tests/test_update_client_carriage.py", "Tlamatini/agent/torrenter_workspace/workspace.html", "Tlamatini/agent/torrenter_workspace/workspace.css", "Tlamatini/agent/torrenter_workspace/layout.js", "Tlamatini/agent/torrenter_workspace/workspace.js", # Terminater: visible terminals (hub, WebSocket, loopback API, pool agent, dock UI, vendored xterm). "Tlamatini/agent/terminal_hub.py", "Tlamatini/agent/terminal_consumer.py", "Tlamatini/agent/terminal_views.py", "Tlamatini/agent/agents/terminater/terminater.py", "Tlamatini/agent/agents/terminater/terminater_engine.py", "Tlamatini/agent/agents/terminater/terminater_conpty.py", "Tlamatini/agent/agents/terminater/terminater_text.py", "Tlamatini/agent/agents/terminater/config.yaml", "Tlamatini/agent/static/agent/js/terminater_dock.js", "Tlamatini/agent/static/agent/css/terminater_dock.css", "Tlamatini/agent/static/agent/vendor/frontend/xterm/lib/xterm.js", "Tlamatini/agent/static/agent/vendor/frontend/xterm/css/xterm.css", "Tlamatini/agent/static/agent/vendor/frontend/xterm-addon-fit/lib/addon-fit.js", "Tlamatini/agent/static/agent/vendor/frontend/xterm-addon-web-links/lib/addon-web-links.js", "Tlamatini/agent/migrations/0215_add_terminater.py", "Tlamatini/agent/migrations/0216_add_chat_agent_terminater_tool.py", "Tlamatini/agent/migrations/0217_add_terminater_demo_prompts.py", "Tlamatini/agent/terminater_testkit.py", "Tlamatini/agent/test_terminater_text.py", "Tlamatini/agent/test_terminater_engine.py", "Tlamatini/agent/test_terminater_hub_api.py", "Tlamatini/agent/test_terminater_pfp.py", "Tlamatini/agent/test_terminater_agent.py", "Tlamatini/agent/test_terminater_integration.py", "scripts/terminater_visible_common.py", "scripts/terminater_visible_battery.py", "scripts/terminater_visible_chat.py", "scripts/terminater_visible_acp.py", "scripts/terminater_visible_pfp.py", "scripts/terminater_visible_all.py", "docs/terminater.md", "docs/prompting-flow-designer.md", "docs/examples/prompting-kickoff.fpmt", # Central models must survive both frozen execution and a self-rebuild. "Tlamatini/agent/agents/model_settings.py", "Tlamatini/agent/services/flow_knowledge.py", "Tlamatini/agent/chat_agent_runtime.py", "Tlamatini/agent/management/commands/check_agent_runtimes.py", "Tlamatini/agent/test_model_settings.py", "Tlamatini/agent/test_model_tag_upgrade.py", "docs/model_configuration.md", # Fatal visual diagnostics must survive both portable copies and self-rebuilds. "Tlamatini/agent/agents/visual_errors.py", "Tlamatini/agent/visual_error_reporting.py", "Tlamatini/agent/test_visual_fatal_errors.py", "Tlamatini/agent/migrations/0210_strict_visual_analysis_prompt_contract.py", "scripts/visual_fatal_dialog_visible.py", "docs/visual-analysis-errors.md", # Context Governor: the real-request meter and the always-on chat gauge. "Tlamatini/agent/test_usage.py", "docs/usage-dashboard.md", "scripts/usage_dashboard_visible.py", "scripts/usage_dashboard_browser_checks.py", "scripts/usage_dashboard_fixture_checks.py", "Tlamatini/agent/usage_tracking.py", "Tlamatini/agent/usage_provider.py", "Tlamatini/agent/usage_views.py", "Tlamatini/agent/migrations/0213_usage_ledger.py", "Tlamatini/agent/templates/agent/usage_dialog.html", "Tlamatini/agent/static/agent/js/usage_dashboard.js", "Tlamatini/agent/static/agent/css/usage_dashboard.css", "Tlamatini/agent/context_governor.py", "Tlamatini/agent/context_baseline.py", "Tlamatini/agent/static/agent/js/context_gauge.js", "Tlamatini/agent/static/agent/css/context_gauge.css", # Compact mode (2026-10-01): every request fitted to the model's REAL # window; the capacity dialog/badge; readable answer tables. "Tlamatini/agent/context_fitter.py", "Tlamatini/agent/compact_mode.py", "Tlamatini/agent/migrations/0211_compact_state.py", # The Self-modify switch (2026-10-03). "Tlamatini/agent/migrations/0212_compact_state_self_modify.py", "Tlamatini/agent/static/agent/js/self_modify_switch.js", "Tlamatini/agent/static/agent/js/compact_costs.js", "Tlamatini/agent/static/agent/js/model_capacity.js", "Tlamatini/agent/static/agent/css/model_capacity.css", "Tlamatini/agent/static/agent/js/chat_table_contrast.js", # Skill boundary: ONE validator, ONE redaction definition, and the tools # that count the skills and keep the .gemini mirror in step. "Tlamatini/agent/skills/validation.py", "Tlamatini/agent/skills/redaction.py", "scripts/skill_inventory.py", "scripts/sync_assistant_skills.py", # Every skill links to the visible-execution policy; AGENTS.md is the # Codex twin of CLAUDE.md. "TestsVisiblesAndVisibleExecutionFromClaude2Codex.md", "AGENTS.md", # Project dossier generator (tlamatini_app_summary.pdf + the PPTX). The # modules only work together, so all six are guaranteed as a set. "Tlamatini/agent/doc_generation/complete_project_docs.py", "Tlamatini/agent/doc_generation/dossier_content.py", "Tlamatini/agent/doc_generation/dossier_pdf.py", "Tlamatini/agent/doc_generation/dossier_pptx.py", "Tlamatini/agent/doc_generation/dossier_theme.py", "Tlamatini/agent/doc_generation/dossier_verify.py", # Build + packaging pipeline "build.py", "build_runtime_assets.py", "build_installer.py", "build_uninstaller.py", "build_complete_public_release.py", "build_complete_private_release.py", "build_all.cmd", "install.py", "uninstall.py", "regen_secrets.py", "versioning.py", "pyinstaller_guard.py", "build_env.py", "requirements.txt", "copy_source_assets.py", # Self-update capability (in-app checker + external file-swap script) "apply_update.ps1", "Tlamatini/agent/self_update.py", "preserved_user_state.json", "Tlamatini/agent/sqlite_copy.py", "Tlamatini/agent/external_mcp_defaults.py", "Tlamatini/agent/runtime_provisioner.py", ".claude/skills/tlamatini-self-modify-inclusion/SKILL.md", ".claude/skills/tlamatini-self-modify-inclusion/scripts/sweep_self_modify.py", ".claude/skills/tlamatini-self-update-inclusion/SKILL.md", ".claude/skills/tlamatini-self-update-inclusion/scripts/sweep_self_update.py", ".gemini/skills/tlamatini-self-modify-inclusion/SKILL.md", ".gemini/skills/tlamatini-self-modify-inclusion/scripts/sweep_self_modify.py", ".gemini/skills/tlamatini-self-update-inclusion/SKILL.md", ".gemini/skills/tlamatini-self-update-inclusion/scripts/sweep_self_update.py", # PDF canvas: source, native packaging and representative vendor assets. "pyinstaller_hooks/hook-pymupdf.py", "scripts/vendor_pdfjs.py", "scripts/vendor_frontend.py", "Tlamatini/agent/static/agent/js/agent_page_markdown.js", "Tlamatini/agent/static/agent/js/agent_page_editing.js", "Tlamatini/agent/static/agent/css/canvas_editor.css", "Tlamatini/agent/static/agent/vendor/canvas-editor/editor.js", "Tlamatini/agent/static/agent/vendor/canvas-editor/LICENSES.txt", "Tlamatini/agent/static/agent/vendor/canvas-editor/manifest.json", "scripts/canvas_editor_entry.mjs", "scripts/build_canvas_editor.mjs", "scripts/canvas_editing_visible.py", "docs/canvas-editing.md", "Tlamatini/agent/interactor/__init__.py", "Tlamatini/agent/interaction_settings.py", "Tlamatini/agent/interaction_narration.py", "Tlamatini/agent/test_interaction_settings.py", "Tlamatini/agent/migrations/0220_interaction_settings.py", "Tlamatini/agent/templates/agent/interaction_settings.html", "Tlamatini/agent/static/agent/js/interaction_settings.js", "Tlamatini/agent/static/agent/js/interaction_speech.js", "Tlamatini/agent/static/agent/js/voice_runtime.js", "Tlamatini/agent/static/agent/css/interaction_settings.css", "scripts/interaction_settings_visible.py", "docs/interaction-settings.md", "Tlamatini/agent/interactor/context.py", "Tlamatini/agent/interactor/protocol.py", "Tlamatini/agent/interactor/hub.py", "Tlamatini/agent/interactor/consumer.py", "Tlamatini/agent/interactor/files.py", "Tlamatini/agent/interactor/tool.py", "Tlamatini/agent/interactor/editor.mjs", "Tlamatini/agent/static/agent/interactor/bridge.js", "Tlamatini/agent/static/agent/interactor/interactor.css", "Tlamatini/agent/templates/agent/interactor.html", "Tlamatini/agent/migrations/0218_add_interactor_tool.py", "Tlamatini/agent/test_interactor.py", "scripts/interactor_visible.py", "scripts/interactor_runtime_server.py", "scripts/interactor_runtime_visible.py", "scripts/playwright_real_visibility.cjs", "docs/interactor.md", "Tlamatini/agent/static/agent/css/markdown_canvas.css", "Tlamatini/agent/static/agent/vendor/frontend/marked/lib/marked.umd.js", "Tlamatini/agent/static/agent/vendor/frontend/marked/LICENSE", "Tlamatini/agent/static/agent/vendor/frontend/dompurify/dist/purify.min.js", "Tlamatini/agent/static/agent/vendor/frontend/dompurify/LICENSE", "Tlamatini/agent/static/agent/vendor/frontend/manifest.json", "Tlamatini/agent/static/agent/vendor/frontend/bootstrap/dist/css/bootstrap.min.css", "Tlamatini/agent/static/agent/vendor/frontend/bootstrap/dist/js/bootstrap.bundle.min.js", "Tlamatini/agent/static/agent/vendor/frontend/jquery/dist/jquery.min.js", "Tlamatini/agent/pdf_context.py", "Tlamatini/agent/pdf_context_jobs.py", "Tlamatini/agent/pdf_context_views.py", "Tlamatini/agent/pdf_image_analysis.py", "Tlamatini/agent/agents/image_interpreter/image_interpreter.py", "Tlamatini/agent/agents/image_interpreter/config.yaml", "Tlamatini/agent/static/agent/pdf/canvas.html", "Tlamatini/agent/static/agent/js/agent_page_pdf.js", "Tlamatini/agent/static/agent/js/pdf_canvas_viewer.js", "Tlamatini/agent/static/agent/js/pdf_context_progress.js", "Tlamatini/agent/static/agent/css/pdf_canvas.css", "Tlamatini/agent/static/agent/css/pdf_context_progress.css", "Tlamatini/agent/static/agent/vendor/pdfjs/build/pdf.mjs", "Tlamatini/agent/static/agent/vendor/pdfjs/build/pdf.worker.mjs", "Tlamatini/agent/static/agent/vendor/pdfjs/web/pdf_viewer.mjs", "Tlamatini/agent/static/agent/vendor/pdfjs/web/pdf_viewer.css", "Tlamatini/agent/static/agent/vendor/pdfjs/LICENSE", "Tlamatini/agent/static/agent/vendor/pdfjs/VERSION", "Tlamatini/agent/static/agent/vendor/pdfjs/wasm/jbig2.wasm", "Tlamatini/agent/static/agent/vendor/pdfjs/wasm/openjpeg.wasm", "Tlamatini/agent/static/agent/vendor/pdfjs/wasm/qcms_bg.wasm", "Tlamatini/agent/static/agent/vendor/pdfjs/standard_fonts/LiberationSans-Regular.ttf", "docs/pdf-canvas.md", # Core app source + LLM identity "Tlamatini/agent/config.json", "Tlamatini/agent/prompt.pmt", "Tlamatini/agent/Tlamatini.md", "CLAUDE.md", # External MCP client layer. NOTE: external_mcps.json is gitignored USER # STATE (absent in a fresh checkout, redacted if present) — it is NOT a # required source file; requiring it wrongly aborted snapshot generation. "Tlamatini/agent/external_mcp_manager.py", # Always-present capabilities added since the ESPHomer era. Each is the # canonical file of a whole subsystem, so a careless future exclusion fails # loudly instead of shipping a Tlamatini that cannot rebuild them. "Tlamatini/agent/self_healing.py", # self-healing model steps "Tlamatini/agent/cancellation.py", # per-user run-epoch hard cancel "Tlamatini/agent/path_guard.py", # Temp / Templates directory policy "Tlamatini/agent/agent_manifest.py", # companion-app (FlowPills) discovery "Tlamatini/agent/windows_app_registration.py", # HKCU discovery-key writer # Newest agent (Nmapper) — both the script and its template config "Tlamatini/agent/agents/nmapper/nmapper.py", "Tlamatini/agent/agents/nmapper/config.yaml", # KEEP_PATH_GLOBS canaries — shipped binary assets referenced BY NAME that # the extension rules would otherwise drop. Without the avatar frame the # chat portrait renders broken; without the guide screenshot the shipped # HOW_TO_GET_YOUR_*_ASSETS.md walkthroughs lose their images. "Tlamatini/agent/static/agent/img/avatar/eo_mc.jpg", "Tlamatini/agent/agents/whatsapper/guide_assets/whatsapp_01_meta_create_app.jpg", ) # ── Restore manifest — binaries the REBUILD needs but the snapshot omits ───── # Each entry: snapshot-relative target -> where to copy it from at rebuild # time, relative to the INSTALLED Tlamatini root (next to Tlamatini.exe). RESTORE_FROM_INSTALL = { "Tlamatini/jd-cli/jd-cli.jar": "jd-cli/jd-cli.jar", "Tlamatini/agent/static/agent/video/XAIHT-Tlamatini.mp4": "_internal/agent/static/agent/video/XAIHT-Tlamatini.mp4", } # ── Secret redaction (mirrors regen_secrets.py placeholder style) ─────────── # A key is secret-looking when its LAST underscore-segment(s) match one of # these suffixes. Deliberately does NOT match max_tokens / request_timeout / # device_name / chat history fields. _SECRET_KEY_RE = re.compile( r"(?i)^(?:.*[_\-.])?(" r"api_key|apikey|api_hash|api_id|token|bot_token|secret|client_secret|" r"password|passwd|access_key|chat_id|listen_chat|session_string|username" r")$" ) # YAML line: : (comments after the value survive) _YAML_KV_RE = re.compile(r"^(\s*)([A-Za-z0-9_\-.]+)(\s*:\s*)(.*?)(\s*(#.*)?)$") _SKIP_VALUES = {"", "''", '""', "[]", "{}", "null", "~", "None", "0"} def _placeholder(key: str) -> str: """regen_secrets.py-compatible placeholder.""" return f"<{key} goes here>" def _is_secret_key(key: str) -> bool: return bool(_SECRET_KEY_RE.match(key.strip())) def _value_needs_redaction(value: str) -> bool: v = value.strip().strip("'\"") if v in _SKIP_VALUES or value.strip() in _SKIP_VALUES: return False if v.startswith("<") and v.endswith("goes here>"): return False # already a placeholder return True def _redact_json_obj(obj: Any, hits: List[str]) -> Any: """Deep-walk a parsed JSON structure replacing secret-looking values.""" if isinstance(obj, dict): out = {} for k, v in obj.items(): if (_is_secret_key(str(k)) and v is not None and _value_needs_redaction(str(v))): out[k] = _placeholder(str(k)) hits.append(str(k)) else: out[k] = _redact_json_obj(v, hits) return out if isinstance(obj, list): return [_redact_json_obj(item, hits) for item in obj] return obj def _redact_json_text(text: str) -> tuple[str, int]: """Redact a JSON document; malformed input must abort snapshot generation.""" data = json.loads(text) hits: List[str] = [] redacted = _redact_json_obj(data, hits) return json.dumps(redacted, indent=2, ensure_ascii=False) + "\n", len(hits) def _redact_yaml_text(text: str) -> tuple[str, int]: """Parse nested/flow/block YAML; line regexes can leak continuation lines. PyYAML is a declared build dependency. Never fall back to copying raw text if it is unavailable or the document cannot be parsed safely. """ import yaml data = yaml.safe_load(text) hits: List[str] = [] redacted = _redact_json_obj(data, hits) return yaml.safe_dump(redacted, sort_keys=False, allow_unicode=True), len(hits) # The self-modify snapshot must never carry a user's REAL contacts (phone numbers # / @handles are PII the key-based redactor above does NOT scrub). So contacts.json # is replaced wholesale with this safe example in the snapshot. _CONTACTS_SNAPSHOT_PLACEHOLDER = ( '{\n' ' "_README": "Tlamatini Contacts book. Add people here, then say \'send a ' "WhatsApp / Telegram to '. Fields: name, aliases, telegram (@username " 'for people), whatsapp (+phone), email.",\n' ' "contacts": [\n' ' {"name": "Example Person", "aliases": ["Example"], ' '"telegram": "@example", "whatsapp": "+10000000000", "email": "example@example.com"}\n' ' ]\n' '}\n' ) def _wants_redaction(rel_posix: str) -> str: """Return 'json' / 'yaml' / 'contacts' / '' for files whose secrets must be scrubbed.""" if rel_posix == "Tlamatini/agent/config.json": return "json" if rel_posix == "Tlamatini/agent/external_mcps.json": return "external_mcps" # URLs, args, arbitrary env/header keys may ALL contain credentials if rel_posix == "Tlamatini/agent/contacts.json": return "contacts" # PII (phones / handles) — replace with a safe example if (rel_posix.startswith("Tlamatini/agent/agents/") and rel_posix.endswith("config.yaml")): return "yaml" return "" def validate_snapshot_carriage(repo_root: Path, dest: Path) -> int: """Every runtime build input must be present or explicitly restore-mapped.""" from build_runtime_assets import ROOT_SOURCES, SOURCE_TREES, file_record, tree_files inputs = set(ROOT_SOURCES) # Spanish source rebuilds directly from its translated files. Retain the # corresponding porting inputs as well, so later bilingual maintenance is reproducible. base_file = repo_root / "l10n" / "BASE_INGLES.json" if base_file.is_file(): localization = json.loads(base_file.read_text(encoding="utf-8")) revision = localization.get("parche") if revision: if not re.fullmatch(r"[0-9a-f]{64}", revision): raise RuntimeError("Invalid localization source revision") inputs.add("l10n/bases/" + revision + ".patch") inputs.update(("l10n/BASE_INGLES.json", "l10n/portar.py", "l10n/herramientas/base_ingles.py")) carriers = {name: {destination} for name, destination in ROOT_SOURCES.items()} for src, destination in SOURCE_TREES: if src != "Tlamatini/staticfiles": # collectstatic regenerates this tree for path in tree_files(repo_root / src, source=True): name = path.relative_to(repo_root).as_posix() inputs.add(name) carriers.setdefault(name, set()).add( destination + "/" + path.relative_to(repo_root / src).as_posix()) for name in sorted(inputs): if name in RESTORE_FROM_INSTALL: if not (repo_root / name).is_file(): raise RuntimeError(f"Restore-mapped build input is missing: {name}") if RESTORE_FROM_INSTALL[name] not in carriers[name]: raise RuntimeError(f"Restore mapping has no matching runtime carrier: {name}") continue target = dest / name if not target.is_file(): raise RuntimeError(f"Snapshot dropped a runtime build input: {name}") if not _wants_redaction(name) and file_record(target) != file_record(repo_root / name): raise RuntimeError(f"Snapshot changed a runtime build input: {name}") return len(inputs) # ── Walk + copy ────────────────────────────────────────────────────────────── def _is_kept_by_path(rel_posix: str) -> bool: """True when a snapshot-relative POSIX path is explicitly KEPT (see KEEP_PATH_GLOBS) despite an extension/glob drop rule.""" return any(fnmatch.fnmatch(rel_posix, pat) for pat in KEEP_PATH_GLOBS) def _is_secret_file(name: str) -> bool: """True for credential-bearing files (keys / .env / private keys). Checked in the same NEVER-resurrected tier as EXCLUDED_FILE_NAMES — see the comment on SECRET_FILE_EXTENSIONS for why this must beat every KEEP carve-out.""" if Path(name).suffix.lower() in SECRET_FILE_EXTENSIONS: return True return any(fnmatch.fnmatch(name, pat) for pat in SECRET_FILE_GLOBS) def _is_dropped_by_path(rel_posix: str) -> bool: """True when a snapshot-relative POSIX path is ALWAYS dropped (DROP_PATH_GLOBS).""" return any(fnmatch.fnmatch(rel_posix, pat) for pat in DROP_PATH_GLOBS) def _skip_file(name: str, rel_posix: str = "") -> bool: # Secrets / PII / local state are NEVER resurrected by a KEEP carve-out. if name in EXCLUDED_FILE_NAMES or _is_secret_file(name): return True # Path-anchored run artifacts (chat-test reports) are dropped before KEEP too. if rel_posix and _is_dropped_by_path(rel_posix): return True # A path-based KEEP wins over the extension / name-glob drops, so shipped UI # assets (avatar frames) and shipped-doc screenshots (guide_assets) survive. if rel_posix and _is_kept_by_path(rel_posix): return False if Path(name).suffix.lower() in EXCLUDED_EXTENSIONS: return True return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDED_FILE_GLOBS) def _force_remove_readonly(func, path, _exc_info): """rmtree onerror hook: some caches (notably Go's module cache) write files READ-ONLY on Windows, so clearing a stale snapshot dest can raise WinError 5. Clear the read-only bit and retry so a rebuild is never blocked by leftovers.""" try: os.chmod(path, 0o700) func(path) except Exception: pass def copy_source_assets(repo_root: Path | str = REPO_ROOT, dest: Path | str = DEFAULT_DEST, redact: bool = True) -> Dict[str, Any]: """Build the self-modify source snapshot. Returns a stats dict.""" started = time.time() repo_root = Path(repo_root).resolve() dest = Path(dest).resolve() if dest == repo_root or not dest.is_relative_to(repo_root): raise ValueError("dest must be a dedicated directory strictly inside the repo root") if dest.exists(): if not dest.is_dir() or (any(dest.iterdir()) and not (dest / "_SOURCE_SNAPSHOT_MANIFEST.json").is_file()): raise ValueError("Refusing to replace a nonempty directory without a snapshot manifest") shutil.rmtree(dest, onerror=_force_remove_readonly) if dest.exists(): raise RuntimeError("Could not clear the previous source snapshot; refusing a mixed snapshot") dest.mkdir(parents=True, exist_ok=True) files_copied = 0 files_skipped = 0 bytes_copied = 0 redacted_files: Dict[str, int] = {} errors: List[str] = [] for current_dir, dir_names, file_names in os.walk(repo_root): cur = Path(current_dir) # prune excluded directories in place (and never descend into dest) dir_names[:] = sorted( d for d in dir_names if (d not in EXCLUDED_DIR_NAMES or (cur / d).relative_to(repo_root).as_posix() in KEEP_DIR_PATHS) and d.rstrip(" .").casefold() != "ms-playwright" and d != "_chat_runs_" and (cur / d).resolve() != dest ) for name in dir_names: entry = cur / name if entry.is_symlink() or (hasattr(entry, "is_junction") and entry.is_junction()): raise RuntimeError(f"Source directory is a link, not a snapshot input: {entry.relative_to(repo_root)}") for fname in sorted(file_names): src = cur / fname rel = src.relative_to(repo_root) rel_posix = rel.as_posix() if _skip_file(fname, rel_posix): files_skipped += 1 continue if src.is_symlink() or not src.resolve().is_relative_to(repo_root): raise RuntimeError(f"Source file escapes the snapshot boundary: {rel_posix}") dst = dest / rel try: dst.parent.mkdir(parents=True, exist_ok=True) mode = _wants_redaction(rel_posix) if redact else "" if mode: try: text = src.read_text(encoding="utf-8") if mode == "json": new_text, n = _redact_json_text(text) elif mode == "contacts": new_text, n = _CONTACTS_SNAPSHOT_PLACEHOLDER, 1 elif mode == "external_mcps": # Startup seeds maintained defaults from external_mcp_defaults.py. new_text, n = '{"mcpServers": {}, "active": []}\n', 1 else: new_text, n = _redact_yaml_text(text) except Exception: # Parser exceptions can contain the original secret; do not echo them. raise RuntimeError(f"Cannot safely redact {rel_posix}; snapshot aborted") from None dst.write_text(new_text, encoding="utf-8") if n: redacted_files[rel_posix] = n else: shutil.copy2(src, dst) files_copied += 1 bytes_copied += dst.stat().st_size except OSError as exc: errors.append(f"{rel_posix}: {exc}") if files_copied == 0: raise RuntimeError(f"copy_source_assets produced an EMPTY snapshot at {dest} " f"from {repo_root} — refusing to continue.") # Completeness guarantee — fail loudly if a build-/run-critical source was # dropped (e.g. by a future exclusion rule) rather than ship a broken tree. missing = [rel for rel in REQUIRED_SNAPSHOT_FILES if not (dest / rel).is_file()] if missing: raise RuntimeError( "copy_source_assets snapshot is INCOMPLETE — these required source " "files are missing from " + str(dest) + ":\n - " + "\n - ".join(missing) + "\n(Either the file was removed from the repo, or an exclusion rule " "wrongly dropped it — fix before shipping a self-modify build.)" ) runtime_inputs_verified = validate_snapshot_carriage(repo_root, dest) if errors: raise RuntimeError("Snapshot copy failed for: " + ", ".join(errors)) stats: Dict[str, Any] = { "generated_at": datetime.now().isoformat(timespec="seconds"), "repo_root": str(repo_root), "dest": str(dest), "files_copied": files_copied, "files_skipped": files_skipped, "bytes_copied": bytes_copied, "megabytes_copied": round(bytes_copied / 1048576, 2), "files_redacted": len(redacted_files), "runtime_inputs_verified": runtime_inputs_verified, "redactions": redacted_files, "restore_from_install": RESTORE_FROM_INSTALL, "errors": errors, "seconds": round(time.time() - started, 1), } (dest / "_SOURCE_SNAPSHOT_MANIFEST.json").write_text( json.dumps({k: v for k, v in stats.items() if k not in {"repo_root", "dest"}}, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") (dest / "_REBUILD_INSTRUCTIONS.md").write_text( _rebuild_instructions(), encoding="utf-8") return stats def _rebuild_instructions() -> str: restore_lines = "\n".join( f" - copy `/{src}` -> `{dst}` (under this directory)" for dst, src in RESTORE_FROM_INSTALL.items() ) return f"""# How to rebuild Tlamatini from this source snapshot This directory is a COMPLETE source snapshot generated by `copy_source_assets.py` (omitting media, secrets, and binaries already present in the installed tree). `` below = the installed Tlamatini directory (where Tlamatini.exe lives). 1. **Restore the omitted binaries** (they were skipped because the install already carries them): {restore_lines} 2. **Restore live secrets** — `Tlamatini/agent/config.json` (and a few agent `config.yaml` files) here carry `` placeholders. Copy the real values from `/config.json` (and the installed agents' configs), or run `python regen_secrets.py --mode keyed` with a populated `data.keys`. 3. **Choose a build version** before building. This snapshot excludes `.git` and generated `_version.py`, so it cannot recover a version from Git history. Set `$env:TLAMATINI_VERSION = ''` in the PowerShell session used for all build commands. Use an intentional version so About / Check for updates can compare it correctly; do not ship `0.0.0+unknown`. 4. **Install build deps** into a Python 3.12.10 interpreter: `python -m pip install -r requirements.txt` (build.py also re-installs them). 5. **Regenerate static files** if needed: `python Tlamatini/manage.py collectstatic --noinput` (`build.py` also runs this with `--clear`). Frontend/PDF.js vendor assets are carried locally with receipts; no CDN is required at runtime. 6. **Build**: `python build.py --self-modify` builds the application and its matching removal client. Run `python build_installer.py` to assemble the release. The `--self-modify` flag re-runs `copy_source_assets.py`, so the NEW build's snapshot automatically reflects every modification made here. 7. **Complete release**: use `build_complete_public_release.py --self-modify` (or the private counterpart for a deliberately private release). Both check snapshot/identity mode against `runtime-assets.json` and enforce the final OUTER ZIP limit of 1,990,000,000 bytes. Do not delete runtime files to fit. Reconfigure private external MCP servers: their local catalog is reset in this source snapshot, not copied with embedded credentials. """ def main(argv: List[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) parser.add_argument("--repo-root", default=str(REPO_ROOT), help="Source repository root (default: this script's directory)") parser.add_argument("--dest", default=str(DEFAULT_DEST), help=f"Snapshot destination (default: {DEFAULT_DEST})") parser.add_argument("--no-redact", action="store_true", help="Keep secret values verbatim (LOCAL USE ONLY)") args = parser.parse_args(argv) stats = copy_source_assets(args.repo_root, args.dest, redact=not args.no_redact) print(f"Snapshot: {stats['dest']}") print(f" files copied : {stats['files_copied']}") print(f" files skipped : {stats['files_skipped']}") print(f" size : {stats['megabytes_copied']} MB") print(f" files redacted : {stats['files_redacted']}") if stats["errors"]: print(f" ERRORS ({len(stats['errors'])}):") for e in stats["errors"]: print(f" - {e}") return 1 return 0 if __name__ == "__main__": sys.exit(main())