#!/usr/bin/env python3
"""
AUTONOMY PRODUCTION v1.0 - Diverse Exploration + Self-Improving
Integration of strategy_factory, scheduler, evaluator, reports
Robust polling + MINI SMOKE GATE
"""

import json
import subprocess
import zipfile
import time
import random
import hashlib
import sys
import os
import shutil
from pathlib import Path
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading

# ============ PYTHON MODULE INTEGRATION ============
# Relative imports from workspace structure
import sys
from pathlib import Path

# Add parent directories to path for imports
# __file__ is in orchestration/loops/, so we need to go up 2 levels to repo root
SCRIPT_DIR = Path(__file__).parent.parent.parent  # bob_der_botmeister/
sys.path.insert(0, str(SCRIPT_DIR))
sys.path.insert(0, str(SCRIPT_DIR / "modules"))

# Import determinism first to ensure seeding happens before any random operations
from utils.seeding.determinism import seed_all, deterministic_hash, SeededRNG
_SEED = seed_all()  # Initialize determinism globally

from modules.strategy_factory import StrategyFactory
from modules.scheduler import DiversityScheduler
from modules.evaluator import Evaluator
from modules.reports.summary import SummaryReporter
from modules.reports.diversity import DiversityReporter
from modules.reports.integrity import IntegrityReporter
from modules.reports.next_batch import NextBatchPlanner
from core.pair_policy import PairPolicy, PairsetConfig

# ============ CONFIGURATION ============
BASE_DIR = Path(SCRIPT_DIR) / "autonomy_runs"
TODAY = datetime.now().strftime("%Y%m%d")
BATCH_ID = f"{TODAY}_autonomy_v1"
OUTPUT_DIR = BASE_DIR / BATCH_ID
RUNS_DIR = OUTPUT_DIR / "runs"
FREQTRADE_BASE = Path("/opt/docker/freqtrade/shared_data/user_data")
STRATEGIES_DIR = FREQTRADE_BASE / "strategies"
BACKTEST_BASE = FREQTRADE_BASE / "backtest_results"
DOCKER_ENV = {"DOCKER_HOST": "tcp://socket-proxy:2375", "PATH": "/usr/bin:/bin:/usr/local/bin:/usr/sbin"}

MAX_WORKERS = 6  # Conservative for stability
BATCH_SIZE = 20  # Checkpoint every N runs
SMOKE_GATE_RUNS = 3  # Mini smoke gate before full batch

# Initialize Pair Policy
PAIR_POLICY = PairPolicy(Path(SCRIPT_DIR) / "data" / "pairlists" / "pairs_manifest.json")

# Timerange options (with pair policy awareness)
# Note: Pair selection will be dynamic based on these
TIMERANGE_OPTIONS = [
    ("20200101-20201231", "2020_full"),
    ("20210101-20211231", "2021_full"),
    ("20220101-20221231", "2022_bear"),
    ("20230101-20231231", "2023_recovery"),
    ("20240101-20240630", "2024_h1"),
    ("20240701-20241231", "2024_h2"),
    ("20240101-20241231", "2024_full"),
]

print_lock = threading.Lock()
anomalies = {"identical": [], "missing_zip": 0, "parse_fallback": 0, "total_completed": 0}
metric_signatures = {}
factory = StrategyFactory(seed=_SEED)
scheduler = DiversityScheduler(seed=_SEED)
evaluator = Evaluator()

# Pair policy allocation tracking
pairset_allocation = {
    "core": 0,
    "core_extended": 0,
    "novelty": 0
}

def log(msg, level="INFO"):
    with print_lock:
        ts = datetime.now().strftime("%H:%M:%S")
        print(f"[{ts}] [{level}] {msg}")
        sys.stdout.flush()

def select_pairs_pair_policy(run_id: str, timerange: str, count: int, mode: str) -> PairsetConfig:
    """Select pairs using Pair Policy with timerange-aware eligibility"""
    seed = hash(run_id) % 2**32
    return PAIR_POLICY.select_pairs_for_run(run_id, timerange, count, mode, seed)

def compute_signature(metrics: dict) -> str:
    """Compute metrics signature for anomaly detection"""
    return f"{metrics.get('trades')}_{metrics.get('profit_pct')}_{metrics.get('pf')}_{metrics.get('max_dd')}"

def check_anomalies(result: dict):
    """Check for anomalies"""
    global anomalies, metric_signatures
    
    if result.get("status") != "SUCCESS":
        if result.get("error") == "No ZIP":
            anomalies["missing_zip"] += 1
        return
    
    sig = compute_signature(result)
    if sig in metric_signatures:
        anomalies["identical"].append({
            "run1": metric_signatures[sig],
            "run2": result["run_id"],
            "signature": sig
        })
        log(f"🚨 IDENTICAL METRICS: {metric_signatures[sig]} vs {result['run_id']}", "CRITICAL")
    else:
        metric_signatures[sig] = result["run_id"]

def robust_wait_for_zip(scan_dir: Path, pattern: str, run_id: str, 
                        max_wait: int = 900, run_dir: Path = None) -> Path:
    """
    Robust polling for ZIP file with:
    - Adaptive backoff (3s -> 5s -> 8s -> 15s)
    - File stability check (unchanged size across 2 polls)
    - Docker log capture on timeout
    """
    poll_intervals = [3, 5, 8, 15]  # Adaptive backoff
    poll_idx = 0
    wait_time = 0
    
    last_size = 0
    stable_count = 0
    
    while wait_time < max_wait:
        # Search for matching files
        matches = list(scan_dir.glob(pattern))
        
        if matches:
            # Check newest ZIP
            zip_path = max(matches, key=lambda p: p.stat().st_mtime)
            current_size = zip_path.stat().st_size
            
            # File stable check
            if current_size == last_size and current_size > 1000:
                stable_count += 1
                if stable_count >= 2:  # Stable for 2 consecutive checks
                    return zip_path
            else:
                stable_count = 0
            
            last_size = current_size
        
        # Sleep with adaptive backoff
        sleep_time = poll_intervals[min(poll_idx, len(poll_intervals)-1)]
        time.sleep(sleep_time)
        wait_time += sleep_time
        poll_idx += 1
    
    # Timeout - collect docker logs if run_dir provided
    if run_dir:
        log(f"⏱ TIMEOUT waiting for {run_id}, collecting logs...", "WARN")
        try:
            # Get recent container logs
            result = subprocess.run(
                ["docker", "logs", "--tail", "200", f"freqtrade-{run_id[:30]}"],
                capture_output=True, text=True, timeout=30,
                env={**os.environ, **DOCKER_ENV}
            )
            with open(run_dir / "docker_tail.log", 'w') as f:
                f.write(result.stdout)
                f.write("\n--- STDERR ---\n")
                f.write(result.stderr)
        except Exception as e:
            with open(run_dir / "docker_tail.log", 'w') as f:
                f.write(f"Failed to collect logs: {e}")
    
    return None

def generate_strategy(run_id: str, params: dict, family: str, timeframe: str) -> str:
    """Generate strategy code for any family"""
    
    # Common imports
    imports = """from freqtrade.strategy import IStrategy
from pandas import DataFrame
import talib.abstract as ta
import numpy as np
"""
    
    # Common class setup
    stoploss = params.get('stoploss', -0.05)
    trailing = params.get('trailing', 0.016)
    trailing_offset = params.get('trailing_offset', 0.025)
    
    class_def = f"""class {run_id}(IStrategy):
    timeframe = '{timeframe}'
    stoploss = {stoploss}
    minimal_roi = {{"0": 1.0}}
    trailing_stop = True
    trailing_stop_positive = {trailing}
    trailing_stop_positive_offset = {trailing_offset}
    trailing_only_offset_is_reached = False
    max_open_trades = 5
    
"""
    
    # Family-specific indicators and logic
    if family in ["breakout_uptrend", "trend_pullback", "hybrid_trend_mr"]:
        indicators = """    def populate_indicators(self, df, m):
        df['adx'] = ta.ADX(df, timeperiod=14)
        df['ema50'] = ta.EMA(df, timeperiod=50)
        df['ema200'] = ta.EMA(df, timeperiod=200)
        df['rsi'] = ta.RSI(df, timeperiod=14)
        try:
            bb = ta.BBANDS(df, nbdevup=2, nbdevdn=2, timeperiod=20)
            df['bb_upper'] = bb['upperband']
            df['bb_middle'] = bb['middleband']
            df['bb_lower'] = bb['lowerband']
        except:
            df['bb_middle'] = df['close'].rolling(20).mean()
            df['bb_upper'] = df['bb_middle'] + 2 * df['close'].rolling(20).std()
            df['bb_lower'] = df['bb_middle'] - 2 * df['close'].rolling(20).std()
        df['vol_ma'] = df['volume'].rolling(20).mean()
        df['vol_spike'] = df['volume'] / df['vol_ma'].replace(0, 1)
        return df
"""
        entry_logic = f"""    def populate_buy_trend(self, df, m):
        adx = {params.get('adx_min', 25)}
        breakout = {params.get('breakout_mult', 1.0)}
        vol = {params.get('volume_mult', 1.0)}
        df['buy'] = 0
        c1 = df['adx'] > adx
        c2 = df['close'] > df['bb_upper'] * breakout
        c3 = df['vol_spike'] > vol
        c4 = df['close'] > df['ema50'] * 0.98
        df.loc[c1 & c2 & c3 & c4, 'buy'] = 1
        return df
"""
    elif family in ["range_mr_long", "range_mr_short"]:
        indicators = """    def populate_indicators(self, df, m):
        df['rsi'] = ta.RSI(df, timeperiod=14)
        df['ema20'] = ta.EMA(df, timeperiod=20)
        df['ema100'] = ta.EMA(df, timeperiod=100)
        try:
            bb = ta.BBANDS(df, nbdevup=2, nbdevdn=2, timeperiod=20)
            df['bb_upper'] = bb['upperband']
            df['bb_lower'] = bb['lowerband']
        except:
            df['bb_lower'] = df['close'].rolling(20).mean() - 2 * df['close'].rolling(20).std()
            df['bb_upper'] = df['close'].rolling(20).mean() + 2 * df['close'].rolling(20).std()
        return df
"""
        rsi_entry = params.get('rsi_entry', 35)
        rsi_exit = 55
        entry_logic = f"""    def populate_buy_trend(self, df, m):
        df['buy'] = 0
        rsi = {rsi_entry}
        c1 = df['rsi'] < rsi
        c2 = df['ema100'] > df['ema100'].shift(1)  # Uptrend
        c3 = df['close'] < df['bb_lower'] * 1.02
        df.loc[c1 & c2 & c3, 'buy'] = 1
        return df
"""
    else:
        # Default breakout strategy for all other families
        indicators = """    def populate_indicators(self, df, m):
        df['adx'] = ta.ADX(df, timeperiod=14)
        df['ema50'] = ta.EMA(df, timeperiod=50)
        df['rsi'] = ta.RSI(df, timeperiod=7)
        df['vol_ma'] = df['volume'].rolling(20).mean()
        df['vol_spike'] = df['volume'] / df['vol_ma'].replace(0, 1)
        return df
"""
        entry_logic = """    def populate_buy_trend(self, df, m):
        df['buy'] = 0
        c1 = df['adx'] > 25
        c2 = df['close'] > df['ema50']
        c3 = df['vol_spike'] > 1.2
        c4 = df['rsi'] > 45
        df.loc[c1 & c2 & c3 & c4, 'buy'] = 1
        return df
"""
    
    exit_logic = """    def populate_sell_trend(self, df, m):
        df['sell'] = 0
        # Use trailing stop, minimal sell signals
        return df
"""
    
    return imports + class_def + indicators + entry_logic + exit_logic

def run_single(run_spec: dict) -> dict:
    """Execute single backtest with robust polling"""
    run_id = run_spec["run_id"]
    family = run_spec["family"]
    timeframe = run_spec["timeframe"]
    timerange = run_spec["timerange"]
    paircount = run_spec["paircount"]
    params = run_spec["params"]
    pairset_mode = run_spec.get("pairset_mode", "core")  # NEW: Pair policy mode
    
    # Create run directory
    run_dir = RUNS_DIR / run_id
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / "config").mkdir(exist_ok=True)
    (run_dir / "params").mkdir(exist_ok=True)
    (run_dir / "pairset").mkdir(exist_ok=True)
    (run_dir / "artifacts").mkdir(exist_ok=True)
    (run_dir / "results").mkdir(exist_ok=True)
    
    result = {"run_id": run_id, "family": family, "status": "INIT"}
    
    try:
        # Select pairs using Pair Policy
        pairset_config = select_pairs_pair_policy(run_id, timerange, paircount, pairset_mode)
        pairs = pairset_config.pairs
        
        # Generate strategy
        strategy_code = generate_strategy(run_id, params, family, timeframe)
        strategy_path = STRATEGIES_DIR / f"{run_id}.py"
        with open(strategy_path, 'w') as f:
            f.write(strategy_code)
        
        # Write config snapshot with Pair Policy info
        config_snapshot = {
            "run_id": run_id, "family": family, "timeframe": timeframe,
            "timerange": timerange, "pair_count": paircount, "pairs": pairs,
            "timestamp_start": datetime.now().isoformat(),
            "pair_policy": {
                "version": pairset_config.policy_version,
                "mode": pairset_config.mode,
                "pairset_id": pairset_config.pairset_id,
                "selection_reason": pairset_config.selection_reason
            }
        }
        with open(run_dir / "config" / "snapshot.json", 'w') as f:
            json.dump(config_snapshot, f, indent=2, default=str)
        
        # Write params
        with open(run_dir / "params" / "params.json", 'w') as f:
            json.dump(params, f, indent=2, default=str)
        
        # Write pairset
        with open(run_dir / "pairset" / "pairs.json", 'w') as f:
            json.dump(pairs, f, indent=2)
        
        # Write config to strategies dir (already mounted)
        config_json = {
            "max_open_trades": paircount,
            "stake_currency": "USDT",
            "stake_amount": 100,
            "dry_run": True,
            "timeframe": timeframe,
            "exchange": {"name": "binance", "pair_whitelist": pairs},
            "pair_whitelist": pairs,
            "data": {"download_trades": False}
        }
        config_path = STRATEGIES_DIR / f"{run_id}_config.json"
        with open(config_path, 'w') as f:
            json.dump(config_json, f)
        
        # Docker command - config in strategies dir
        export_name = f"run_{run_id}"
        cmd = [
            "docker", "run", "--rm",
            "-v", f"{STRATEGIES_DIR}:/freqtrade/user_data/strategies",
            "-v", f"{run_dir}/artifacts:/freqtrade/user_data/backtest_results",
            "-v", f"{FREQTRADE_BASE}/data:/freqtrade/user_data/data:ro",
            "freqtradeorg/freqtrade:stable",
            "backtesting",
            "--config", f"/freqtrade/user_data/strategies/{run_id}_config.json",
            "--strategy", run_id,
            "--timeframe", timeframe,
            "--timerange", timerange,
            "--export", "trades",
            "--export-filename", export_name,
            "--pairs"
        ] + pairs
        
        # Execute with 5 minute timeout
        env = {**os.environ, **DOCKER_ENV}
        start_time = time.time()
        
        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)
        exec_time = time.time() - start_time
        
        # Save logs
        with open(run_dir / "results" / "stdout.txt", 'w') as f:
            f.write(proc.stdout)
        with open(run_dir / "results" / "stderr.txt", 'w') as f:
            f.write(proc.stderr)
        
        # ROBUST WAIT for ZIP with polling
        zip_path = robust_wait_for_zip(
            run_dir / "artifacts",
            f"{export_name}*.zip",
            run_id,
            max_wait=900,
            run_dir=run_dir
        )
        
        if not zip_path:
            result.update({"status": "FAILED", "error": "timeout_waiting_for_zip"})
            # Save minimal metrics
            with open(run_dir / "metrics.json", 'w') as f:
                json.dump(result, f)
            return result
        
        # Parse from ZIP only
        with zipfile.ZipFile(zip_path, 'r') as zf:
            json_files = [f for f in zf.namelist() if f.endswith('.json') and '_config' not in f]
            if not json_files:
                result.update({"status": "FAILED", "error": "No JSON in ZIP"})
                return result
            
            with zf.open(json_files[0]) as f:
                data = json.load(f)
            
            if 'strategy' not in data:
                result.update({"status": "FAILED", "error": "No strategy data"})
                return result
            
            strat = list(data['strategy'].keys())[0]
            s = data['strategy'][strat]
            
            # Extract metrics
            metrics = s.get('results', s) if 'results' in s else s
            trades = metrics.get('total_trades', 0)
            profit = (metrics.get('profit_total') or 0) * 100
            pf = metrics.get('profit_factor', 0)
            dd = (metrics.get('max_drawdown_account') or 0) * 100
            
            # Calculate TPPPD
            dur_days = 365
            try:
                years = int(timerange.split('-')[1][:4]) - int(timerange.split('-')[0][:4]) + 1
                dur_days = years * 365
            except:
                pass
            tpppd = (trades / paircount) / dur_days if trades > 0 else 0
            
            result.update({
                "status": "SUCCESS",
                "trades": trades,
                "profit_pct": round(profit, 2),
                "pf": round(pf, 2),
                "max_dd": round(dd, 2),
                "tpppd": round(tpppd, 4),
                "high_churn": tpppd > 0.5,
                "exec_time": round(exec_time, 1),
                "artifact": zip_path.name
            })
            
            # Evaluate with Autonomy labels
            eval_result = evaluator.evaluate(run_id, result, config=config_snapshot)
            result["label"] = eval_result.label
            result["label_reasoning"] = eval_result.reasoning
            
    except subprocess.TimeoutExpired:
        result.update({"status": "TIMEOUT", "error": "Docker timeout (300s)"})
    except Exception as e:
        result.update({"status": "FAILED", "error": str(e)[:200]})
    
    # Save metrics
    with open(run_dir / "metrics.json", 'w') as f:
        json.dump(result, f, indent=2)
    
    # Save manifest fragment - includes Pair Policy info
    with open(run_dir / "manifest_fragment.json", 'w') as f:
        json.dump({
            "run_id": run_id,
            "config": config_snapshot,  # Now includes pair_policy info!
            "params": params,
            "results": result,
            "timestamp": datetime.now().isoformat()
        }, f, indent=2, default=str)
    
    return result

def generate_seed(run_id: str) -> int:
    """Deterministic seed"""
    return int(hashlib.md5(run_id.encode()).hexdigest(), 16) % 1000000

def load_previous_teachers():
    """Load TEACHER_ALIVE_2024 from previous production run"""
    from modules.strategy_factory import StrategyConfig
    
    teacher_file = Path(SCRIPT_DIR) / "teacher_analysis" / "teacher_param_pack.json"
    if teacher_file.exists():
        with open(teacher_file) as f:
            data = json.load(f)
        
        for run_id, info in data.get('runs', {}).items():
            if info.get('label') in ['TEACHER_ALIVE_2024', 'POSITIVE_WEAK', 'POSITIVE']:
                # Create proper StrategyConfig object
                config = StrategyConfig(
                    family=info['config']['family'],
                    timeframe=info['config']['timeframe'],
                    params=info['params'],
                    config_hash=info.get('config', {}).get('config_hash', 'unknown')
                )
                scheduler.add_exploit_candidate(
                    run_id=run_id,
                    config=config,
                    metrics=info['metrics'],
                    label=info['label']
                )
        log(f"Loaded {len(scheduler.exploit_candidates)} teacher candidates")

def create_batch_plan(total_runs: int) -> list:
    """Create batch plan using scheduler + Pair Policy allocation"""
    plan = scheduler.create_batch_plan(total_runs)
    validation = scheduler.validate_plan(plan)
    
    log(f"\nBatch Plan: {plan.total_runs} runs")
    log(f"  Explore: {plan.explore_runs} ({plan.explore_runs/plan.total_runs*100:.0f}%)")
    log(f"  Exploit: {plan.exploit_runs} ({plan.exploit_runs/plan.total_runs*100:.0f}%)")
    log(f"  Novelty: {plan.novelty_runs} ({plan.novelty_runs/plan.total_runs*100:.0f}%)")
    log(f"  Distinct families: {validation['distinct_families']}")
    
    # Allocate pairset modes: 60% core, 25% core+extended, 15% novelty
    pairset_modes = PAIR_POLICY.allocate_modes(total_runs)
    log(f"  Pairset distribution: {pairset_allocation}")
    
    # Convert to run specs
    run_specs = []
    for i, (config, source) in enumerate(zip(plan.configs, plan.sources)):
        run_id = f"{BATCH_ID}_{config.family[:4]}_{i:04d}"
        pairset_mode = pairset_modes[i]  # Get assigned mode for this run
        
        # Select timerange (ensure diversity, not all 2024)
        seed = generate_seed(run_id)
        rng = random.Random(seed)
        
        # 25% non-2024 for robustness
        if i % 4 == 0:
            # Non-2024
            timerange = rng.choice([tr for tr, _ in TIMERANGE_OPTIONS if '2024' not in tr])
        elif i % 4 == 1:
            # Half year
            timerange = rng.choice([tr for tr, _ in TIMERANGE_OPTIONS if '_h1' in tr or '_h2' in tr])
        else:
            # Full 2024 (safe zone)
            timerange = "20240101-20241231"
        
        # Vary paircount
        paircount = rng.choice([10, 15, 20, 30])
        
        run_specs.append({
            "run_id": run_id,
            "family": config.family,
            "timeframe": config.timeframe,
            "timerange": timerange,
            "paircount": paircount,
            "params": config.params,
            "source": source,
            "config_hash": config.config_hash,
            "pairset_mode": pairset_mode  # NEW: Pair Policy mode
        })
    
    return run_specs

def run_batch(run_specs: list, checkpoint_every: int = 20):
    """Run batch with checkpoints"""
    total = len(run_specs)
    completed = 0
    results = []
    
    log(f"\nStarting batch: {total} runs, {MAX_WORKERS} workers")
    
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(run_single, spec): spec for spec in run_specs}
        
        for future in as_completed(futures):
            spec = futures[future]
            try:
                result = future.result(timeout=900)
                results.append(result)
                check_anomalies(result)
                completed += 1
                
                status = "✓" if result.get("status") == "SUCCESS" else "✗"
                pf = result.get("pf", 0)
                label = result.get("label", "N/A")
                log(f"{status} {spec['run_id']}: PF={pf:.2f}, {label} ({completed}/{total})")
                
                # Checkpoint every N runs
                if completed % checkpoint_every == 0:
                    write_checkpoint(results, completed, total)
                    
            except Exception as e:
                log(f"✗ {spec['run_id']}: Exception {e}", "ERROR")
                results.append({"run_id": spec["run_id"], "status": "FAILED", "error": str(e)})
    
    return results

def write_checkpoint(results: list, completed: int, total: int):
    """Write checkpoint summary"""
    successful = [r for r in results if r.get("status") == "SUCCESS"]
    labels = {}
    for r in successful:
        lbl = r.get("label", "UNKNOWN")
        labels[lbl] = labels.get(lbl, 0) + 1
    
    lines = [
        f"# Checkpoint: {completed}/{total}",
        f"**Timestamp:** {datetime.now().isoformat()}",
        "",
        "## Progress",
        f"- Completed: {completed}/{total} ({completed/total*100:.1f}%)",
        f"- Successful: {len(successful)}",
        "",
        "## Label Distribution",
    ]
    for lbl, cnt in sorted(labels.items(), key=lambda x: -x[1]):
        lines.append(f"- {lbl}: {cnt}")
    
    # Top performers
    top = sorted(successful, key=lambda x: x.get("pf", 0), reverse=True)[:5]
    lines.extend(["", "## Top 5 by PF"])
    for r in top:
        lines.append(f"- {r['run_id']}: PF={r.get('pf',0):.2f}, Profit={r.get('profit_pct',0):.1f}%")
    
    # Anomalies
    lines.extend(["", "## Anomalies"])
    lines.append(f"- Identical metrics: {len(anomalies['identical'])}")
    lines.append(f"- Missing ZIPs: {anomalies['missing_zip']}")
    
    checkpoint_path = OUTPUT_DIR / "checkpoint_summary.md"
    with open(checkpoint_path, 'w') as f:
        f.write('\n'.join(lines))
    
    log(f"Checkpoint saved: {checkpoint_path}")

def generate_reports(all_results: list):
    """Generate all final reports"""
    log("\nGenerating reports...")
    
    # Build manifest
    manifest = {
        "batch_id": BATCH_ID,
        "timestamp": datetime.now().isoformat(),
        "total_runs": len(all_results),
        "successful": sum(1 for r in all_results if r.get("status") == "SUCCESS"),
        "anomalies": {
            "identical": len(anomalies["identical"]),
            "missing_zip": anomalies["missing_zip"]
        },
        "runs": []
    }
    
    # Load full run data
    for result in all_results:
        run_dir = RUNS_DIR / result["run_id"]
        frag_file = run_dir / "manifest_fragment.json"
        if frag_file.exists():
            with open(frag_file) as f:
                manifest["runs"].append(json.load(f))
    
    # Save manifest
    with open(OUTPUT_DIR / "manifest.json", 'w') as f:
        json.dump(manifest, f, indent=2, default=str)
    
    # Generate reports
    SummaryReporter().generate(manifest, OUTPUT_DIR / "summary.md")
    log("  ✓ summary.md")
    
    DiversityReporter().generate(manifest, OUTPUT_DIR / "diversity_report.md")
    log("  ✓ diversity_report.md")
    
    integrity = IntegrityReporter().generate(
        manifest, OUTPUT_DIR, anomalies_collected=[],
        output_path=OUTPUT_DIR / "integrity_report.md"
    )
    log("  ✓ integrity_report.md")
    
    NextBatchPlanner().generate(manifest, OUTPUT_DIR / "next_batch_plan.md")
    log("  ✓ next_batch_plan.md")
    
    # Build teacher dataset
    teachers = []
    for run in manifest["runs"]:
        if run["results"].get("label") in ["TEACHER_ALIVE_2024", "POSITIVE_WEAK", "POSITIVE"]:
            teachers.append({
                "run_id": run["run_id"],
                "label": run["results"]["label"],
                "trades": run["results"]["trades"],
                "profit_pct": run["results"]["profit_pct"],
                "pf": run["results"]["pf"],
                "max_dd": run["results"]["max_dd"],
                "timerange": run["config"]["timerange"]
            })
    teachers.sort(key=lambda x: x["pf"], reverse=True)
    
    with open(OUTPUT_DIR / "teacher_dataset.json", 'w') as f:
        json.dump({"count": len(teachers), "teachers": teachers}, f, indent=2)
    log("  ✓ teacher_dataset.json")
    
    return integrity["critical_pass"]

def run_smoke_gate():
    """MINI SMOKE GATE: 3 runs before full batch - tests different pairset modes"""
    log("\n" + "="*60)
    log(" MINI SMOKE GATE (3 runs - testing pairset modes)")
    log("="*60)
    
    smoke_specs = []
    smoke_families = ["breakout_uptrend", "range_mr_long", "momentum_cont"]
    smoke_timeframes = ["4h", "2h", "6h"]
    smoke_modes = ["core", "core_extended", "novelty"]  # Test all 3 modes
    
    for i, family in enumerate(smoke_families):
        config = factory.generate_by_family(family)
        run_id = f"{BATCH_ID}_SMOKE_{i:02d}"
        
        smoke_specs.append({
            "run_id": run_id,
            "family": family,
            "timeframe": smoke_timeframes[i],
            "timerange": "20240101-20241231",
            "paircount": 15,
            "params": config.params,
            "source": "smoke_test",
            "config_hash": config.config_hash,
            "pairset_mode": smoke_modes[i]  # NEW: Different mode per smoke test
        })
    
    # Run sequentially for smoke (simpler debugging)
    smoke_results = []
    for spec in smoke_specs:
        log(f"\nRunning {spec['run_id']} with mode={spec['pairset_mode']}...")
        result = run_single(spec)
        smoke_results.append(result)
        check_anomalies(result)
        
        status = "✓" if result.get("status") == "SUCCESS" else "✗"
        log(f"{status} {spec['run_id']}: Status={result.get('status')}, PF={result.get('pf',0):.2f}")
    
    # Validate
    success_count = sum(1 for r in smoke_results if r.get("status") == "SUCCESS")
    identical_count = len(anomalies["identical"])
    
    log("\n" + "="*60)
    log(" SMOKE GATE VALIDATION")
    log("="*60)
    log(f"Successful: {success_count}/3")
    log(f"Identical metrics: {identical_count}")
    
    if success_count == 3 and identical_count == 0:
        log("✅ SMOKE PASSED - Proceeding to full batch")
        return True
    else:
        log("❌ SMOKE FAILED - Check logs and fix issues", "CRITICAL")
        # Save smoke results for debugging
        with open(OUTPUT_DIR / "smoke_gate_results.json", 'w') as f:
            json.dump(smoke_results, f, indent=2)
        return False

def main():
    global OUTPUT_DIR, RUNS_DIR
    
    log("="*60)
    log(" AUTONOMY PRODUCTION v1.0")
    log("="*60)
    
    # Setup directories
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    RUNS_DIR.mkdir(parents=True, exist_ok=True)
    
    # Load previous teachers
    load_previous_teachers()
    
    # === MINI SMOKE GATE ===
    smoke_passed = run_smoke_gate()
    if not smoke_passed:
        log("\nSTOP: Smoke gate failed. Review and retry.", "CRITICAL")
        return 1
    
    # === FULL BATCH (120 runs) ===
    log("\n" + "="*60)
    log(" FULL BATCH (120 runs)")
    log("="*60)
    
    # Create plan for remaining 117 runs
    remaining_runs = 120 - SMOKE_GATE_RUNS
    run_specs = create_batch_plan(remaining_runs)
    
    # Run batch
    results = run_batch(run_specs, checkpoint_every=BATCH_SIZE)
    
    # Add smoke results to full results
    # (Fetch from run dirs)
    all_results = []
    for i in range(SMOKE_GATE_RUNS):
        run_id = f"{BATCH_ID}_SMOKE_{i:02d}"
        mf_file = RUNS_DIR / run_id / "metrics.json"
        if mf_file.exists():
            with open(mf_file) as f:
                all_results.append(json.load(f))
    all_results.extend(results)
    
    # Generate reports
    critical_pass = generate_reports(all_results)
    
    # Final stats
    successful = sum(1 for r in all_results if r.get("status") == "SUCCESS")
    teachers = sum(1 for r in all_results if r.get("label") in ["TEACHER_ALIVE_2024", "POSITIVE_WEAK", "POSITIVE"])
    
    log("\n" + "="*60)
    log(" BATCH COMPLETE")
    log("="*60)
    log(f"Total: {len(all_results)} | Successful: {successful} | Teachers: {teachers}")
    log(f"Anomalies: {len(anomalies['identical'])} identical")
    log(f"Output: {OUTPUT_DIR}")
    
    if not critical_pass:
        log("\n⚠️ Critical issues detected - review integrity_report.md", "WARN")
    
    return 0 if critical_pass else 1

if __name__ == "__main__":
    sys.exit(main())