import base64
import ast
import io
import json
import logging
import os
import re
import tarfile
import time
import threading
from concurrent.futures import ThreadPoolExecutor, Future
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, Optional
from fnmatch import fnmatch

import requests
from jsonschema import validate, ValidationError

try:
    import anthropic  # type: ignore
except Exception:  # pragma: no cover
    anthropic = None

# =============================================================================
# LOGGING CONFIGURATION
# =============================================================================

# Create logger for this module
logger = logging.getLogger(__name__)

def log_analysis_start(repo_url: str, job_id: str) -> None:
    """Log the start of repository analysis"""
    logger.info(f"🧠 [ANALYZER] Analysis started - repo={repo_url}, job_id={job_id}")

def log_analysis_step(step: str, job_id: str, **kwargs) -> None:
    """Log analysis step progress"""
    details = ', '.join(f"{k}={v}" for k, v in kwargs.items() if v is not None)
    logger.info(f"🔄 [ANALYZER] [{job_id}] {step} - {details}")

def log_analysis_success(repo_url: str, job_id: str, duration_ms: int, **kwargs) -> None:
    """Log successful analysis completion"""
    details = ', '.join(f"{k}={v}" for k, v in kwargs.items() if v is not None)
    logger.info(f"✅ [ANALYZER] Analysis completed - repo={repo_url}, job_id={job_id}, duration={duration_ms}ms - {details}")

def log_analysis_error(repo_url: str, job_id: str, error: str, duration_ms: int) -> None:
    """Log failed analysis"""
    logger.error(f"❌ [ANALYZER] Analysis failed - repo={repo_url}, job_id={job_id}, duration={duration_ms}ms - Error: {error}")

def log_llm_operation(operation: str, **kwargs) -> None:
    """Log LLM operations"""
    details = ', '.join(f"{k}={v}" for k, v in kwargs.items() if v is not None)
    logger.info(f"🤖 [LLM] {operation} - {details}")

# =============================================================================
# CONFIGURATION AND PATHS
# =============================================================================

BASE_DIR = Path(__file__).resolve().parent.parent
ANALYSIS_DIR = BASE_DIR / "config" / "analysis"
ANALYSIS_DIR.mkdir(parents=True, exist_ok=True)

logger.info(f"📁 [ANALYZER] Analysis directory: {ANALYSIS_DIR}")

ANALYSIS_SCHEMA: Dict[str, Any] = {
    "type": "object",
    "required": [
        "schema_version",
        "repo_url",
        "package_manager",
        "install",
        "codeflash",
        "resources",
        "confidence",
    ],
    "properties": {
        "schema_version": {"type": "string"},
        "repo_url": {"type": "string"},
        "package_manager": {"type": "string"},
        "python_version": {"type": "string"},
        "python_packages": {
            "type": "array",
            "items": {
                "oneOf": [
                    {"type": "string"},
                    {
                        "type": "object",
                        "required": ["name"],
                        "properties": {
                            "name": {"type": "string"},
                            "version_spec": {"type": "string"}
                        }
                    }
                ]
            },
        },
        "system_packages": {
            "type": "array",
            "items": {"type": "string"},
        },
        "install": {
            "type": "object",
            "properties": {
                "pre_install_cmds": {"type": "array", "items": {"type": "string"}},
                "install_cmds": {"type": "array", "items": {"type": "string"}},
                "post_install_cmds": {"type": "array", "items": {"type": "string"}},
            },
        },
        "codeflash": {
            "type": "object",
            "properties": {
                "module_root": {"type": "string"},
                "tests_root": {"type": "string"},
                "formatter_cmds": {"type": "array", "items": {"type": "string"}},
            },
        },
        "tests": {
            "type": "object",
            "properties": {
                "test_command": {"type": "string"},
                "test_framework": {"type": "string", "enum": ["pytest", "unittest", "nose"]},
            },
        },
        "resources": {
            "type": "object",
            "properties": {
                "tier": {"type": "string", "enum": ["small", "medium", "large"]},
                "rationale": {"type": "string"},
            },
        },
        "env": {
            "type": "object",
            "properties": {
                "non_secret_env_vars": {"type": "object"},
            },
        },
        "confidence": {"type": "number", "minimum": 0, "maximum": 1},
        "evidence": {"type": "array", "items": {"type": "string"}},
    },
}

PROMPT_SCHEMA_INSTRUCTIONS = f"""
You are analyzing a Python repository to determine its setup requirements and configuration.
Return ONLY a JSON object that matches this exact schema:

{json.dumps(ANALYSIS_SCHEMA, indent=2)}

Requirements:
1. Return ONLY valid JSON, no markdown, no explanations
2. All required fields must be present
3. Use only allowlisted system packages (common build tools)
4. Use only safe shell commands (no rm, curl downloads, etc.)
5. Set confidence between 0.0-1.0 based on evidence quality
6. Populate python_packages with likely PyPI package names needed to run tests
   - Derive from requirements files, pyproject, setup.* and code imports
   - Prefer canonical PyPI names (e.g., sklearn -> scikit-learn, PIL -> Pillow)
   - When possible include version_spec (e.g., "==1.2.3", ">=2.0")
"""

# Interactive protocol: allow LLM to request specific metadata files/dirs up to 5 turns
INTERACTIVE_INSTRUCTIONS = (
    "You are a repository setup analyst. You must decide if the provided metadata is enough "
    "to produce a valid JSON matching the given schema. If not enough, request ONLY minimal "
    "additional metadata files or directory listings using the following protocol.\n\n"
    "Protocol (strict JSON, no prose):\n"
    "- To request file contents: {\"type\": \"request\", \"action\": \"need_files\", \"paths\": [\"pyproject.toml\", ...]}\n"
    "- To request directory listings: {\"type\": \"request\", \"action\": \"need_dirs\", \"dirs\": [\"requirements\", \".github/workflows\"]}\n"
    "- To return the final result: {\"type\": \"final\", \"analysis\": <JSON conforming to schema>}\n\n"
    "Hard rules:\n"
    "- You must not exceed 5 turns total.\n"
    "- Avoid redundant requests; ask only what is necessary.\n"
    "- Focus on metadata only (requirements, pyproject, setup.*, Pipfile, poetry.lock, uv.lock, README, CI, Dockerfiles).\n"
    "- Do NOT request arbitrary code files unless they are packaging metadata.\n"
    "- Prefer inferring package versions from metadata; include version_spec when available.\n"
)

# =============================================================================
# ANALYSIS JOB TRACKING
# =============================================================================

@dataclass
class AnalysisJob:
    job_id: str
    repo_url: str
    status: str  # pending, running, succeeded, failed
    message: str
    result: Optional[Dict[str, Any]] = None

# File-based job tracking to work across gunicorn workers
ANALYSIS_JOBS_FILE = BASE_DIR / "config" / "analysis_jobs.json"
_jobs_lock = threading.Lock()
_executor = ThreadPoolExecutor(max_workers=2)

def _load_jobs() -> Dict[str, AnalysisJob]:
    """Load jobs from persistent storage"""
    try:
        if not ANALYSIS_JOBS_FILE.exists():
            logger.debug(f"📂 [ANALYZER] No jobs file found at {ANALYSIS_JOBS_FILE}")
            return {}
        
        data = json.loads(ANALYSIS_JOBS_FILE.read_text())
        jobs = {}
        for job_id, job_data in data.items():
            jobs[job_id] = AnalysisJob(
                job_id=job_data["job_id"],
                repo_url=job_data["repo_url"],
                status=job_data["status"],
                message=job_data["message"],
                result=job_data.get("result")
            )
        logger.debug(f"📂 [ANALYZER] Loaded {len(jobs)} jobs from storage")
        return jobs
    except Exception as e:
        logger.warning(f"⚠️ [ANALYZER] Failed to load jobs: {e}")
        return {}

def _save_jobs(jobs: Dict[str, AnalysisJob]) -> None:
    """Save jobs to persistent storage"""
    try:
        data = {}
        for job_id, job in jobs.items():
            data[job_id] = {
                "job_id": job.job_id,
                "repo_url": job.repo_url,
                "status": job.status,
                "message": job.message,
                "result": job.result
            }
        ANALYSIS_JOBS_FILE.write_text(json.dumps(data, indent=2))
        logger.debug(f"💾 [ANALYZER] Saved {len(jobs)} jobs to storage")
    except Exception as e:
        logger.error(f"❌ [ANALYZER] Failed to save jobs: {e}")

def _slug(repo_url: str) -> str:
    parts = repo_url.rstrip("/").split("/")
    org = parts[-2] if len(parts) >= 2 else "org"
    name = parts[-1].replace(".git", "").replace(".", "-")
    slug = f"{org}-{name}"
    logger.debug(f"🏷️ [ANALYZER] Generated slug: {slug} for repo: {repo_url}")
    return slug


def _github_tarball_url(repo_url: str) -> str:
    # https://api.github.com/repos/{org}/{repo}/tarball
    parts = repo_url.rstrip("/").split("/")
    org = parts[-2]
    name = parts[-1].replace(".git", "")
    tarball_url = f"https://api.github.com/repos/{org}/{name}/tarball"
    logger.debug(f"🔗 [ANALYZER] GitHub tarball URL: {tarball_url}")
    return tarball_url


def _list_files_from_tarball(tar_bytes: bytes) -> Dict[str, bytes]:
    """Extract files from GitHub tarball"""
    logger.debug(f"📦 [ANALYZER] Extracting files from tarball ({len(tar_bytes)} bytes)")
    files: Dict[str, bytes] = {}
    try:
        with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:gz") as tar:
            members = tar.getmembers()
            logger.debug(f"📦 [ANALYZER] Found {len(members)} members in tarball")
            
            for member in members:
                if member.isfile() and member.size < 1_000_000:  # 1MB limit
                    # GitHub tarballs have a top-level directory, strip it
                    path_parts = member.name.split("/", 1)
                    if len(path_parts) == 2:
                        relative_path = path_parts[1]
                        try:
                            content = tar.extractfile(member)
                            if content:
                                files[relative_path] = content.read()
                        except Exception as e:
                            logger.warning(f"⚠️ [ANALYZER] Failed to extract {relative_path}: {e}")
                            continue
            
        logger.info(f"📦 [ANALYZER] Successfully extracted {len(files)} files from tarball")
        return files
    except Exception as e:
        logger.error(f"❌ [ANALYZER] Failed to extract tarball: {e}")
        return {}


def _heuristics(files: Dict[str, bytes]) -> Dict[str, Any]:
    """Basic heuristics for package manager and project structure"""
    logger.debug("🔍 [ANALYZER] Running heuristics analysis")
    
    heur: Dict[str, Any] = {"package_manager": "unknown", "python_packages": []}
    
    # Detect package manager
    if "poetry.lock" in files or ("pyproject.toml" in files and b"[tool.poetry]" in files.get("pyproject.toml", b"")):
        heur["package_manager"] = "poetry"
    elif "pyproject.toml" in files and b"[tool.uv]" in files.get("pyproject.toml", b""):
        heur["package_manager"] = "uv"
    elif "Pipfile" in files:
        heur["package_manager"] = "pipenv"
    elif "requirements.txt" in files:
        heur["package_manager"] = "pip"
    
    # Guess module root
    common_src_patterns = ["src/", "lib/", ""]
    for pattern in common_src_patterns:
        potential_modules = [f for f in files.keys() if f.startswith(pattern) and f.endswith("/__init__.py")]
        if potential_modules:
            # Take the shortest path as likely main module
            module_path = min(potential_modules, key=len)
            heur["module_root"] = module_path.rsplit("/", 1)[0] or "."
            break
    else:
        heur["module_root"] = "auto"
    
    # Guess tests root
    test_dirs = [f for f in files.keys() if f.startswith(("tests/", "test/")) and "/" in f]
    if test_dirs:
        heur["tests_root"] = "tests" if any(f.startswith("tests/") for f in test_dirs) else "test"
    else:
        heur["tests_root"] = "auto"
    
    logger.info(f"🔍 [ANALYZER] Heuristics completed - {heur}")
    return heur


def _infer_python_packages_from_files(files: Dict[str, bytes]) -> list[Any]:
    """Infer likely PyPI package names and optional version specs from common config files.

    Best-effort parsing of requirements*.txt, pyproject.toml (PEP 621/Poetry),
    setup.cfg and setup.py. Returns a unique, normalized list of either
    strings (package name) or objects {"name": ..., "version_spec": "==1.2.3"}.
    """
    def _split_name_spec(name: str) -> tuple[str, Optional[str]]:
        candidate = name.strip()
        # Strip environment markers and hashes
        if ";" in candidate:
            candidate = candidate.split(";", 1)[0].strip()
        if " #" in candidate:
            candidate = candidate.split(" #", 1)[0].strip()
        # Drop direct URLs, VCS, and local paths
        if candidate.startswith(("http://", "https://", "git+", "svn+", "hg+", "file:")):
            return "", None
        if candidate.startswith(("./", "../", "/")):
            return "", None
        # Extract version spec if present
        version_spec = None
        for sep in ["===", "==", ">=", "<=", "!=", "~=", ">", "<", "^", "~"]:
            if sep in candidate:
                parts = candidate.split(sep, 1)
                version_spec = sep + parts[1].strip()
                candidate = parts[0].strip()
                break
        # Remove extras e.g. package[extra]
        if "[" in candidate:
            candidate = candidate.split("[", 1)[0].strip()
        # Basic sanity: only typical characters
        if not candidate or not all(c.isalnum() or c in {"-", "_", "."} for c in candidate):
            return "", None
        return candidate, version_spec

    def _from_requirements(text: str) -> list[Any]:
        names: list[Any] = []
        for raw in text.splitlines():
            line = raw.strip()
            if not line or line.startswith(("#", "-r ", "--")):
                continue
            pkg, spec = _split_name_spec(line)
            if pkg:
                names.append({"name": pkg, "version_spec": spec} if spec else pkg)
        return names

    def _from_setup_cfg(text: str) -> list[Any]:
        names: list[Any] = []
        in_section = False
        in_list = False
        for raw in text.splitlines():
            line = raw.rstrip()
            if line.strip().startswith("[") and line.strip().endswith("]"):
                in_section = (line.strip().lower() == "[options]")
                in_list = False
                continue
            if in_section:
                stripped = line.strip()
                if stripped.lower().startswith("install_requires"):
                    in_list = True
                    after = stripped.split("=", 1)[-1].strip()
                    if after:
                        names.extend(_from_requirements(after.replace(",", "\n")))
                    continue
                if in_list:
                    if not stripped or stripped.startswith("["):
                        in_list = False
                        continue
                    names.extend(_from_requirements(stripped))
        return names

    def _from_pyproject(text: str) -> list[Any]:
        names: list[Any] = []
        # PEP 621 style: dependencies = ["pkg", ...]
        import re as _re
        for m in _re.finditer(r"dependencies\s*=\s*\[([\s\S]*?)\]", text, _re.IGNORECASE):
            block = m.group(1)
            for q in _re.finditer(r"['\"]([^'\"]+)['\"]", block):
                pkg, spec = _split_name_spec(q.group(1))
                if pkg:
                    names.append({"name": pkg, "version_spec": spec} if spec else pkg)
        # Poetry style: [tool.poetry.dependencies]\nname = "ver"
        in_poetry = False
        for raw in text.splitlines():
            line = raw.strip()
            if line.startswith("[") and line.endswith("]"):
                in_poetry = (line.lower() == "[tool.poetry.dependencies]" or line.lower().startswith("[tool.poetry.group.") and line.lower().endswith(".dependencies]"))
                continue
            if in_poetry and line and not line.startswith(('#', '[')) and "=" in line:
                key = line.split("=", 1)[0].strip().strip('"').strip("'")
                if key.lower() == "python":
                    continue
                pkg, spec = _split_name_spec(key)
                if pkg:
                    names.append({"name": pkg, "version_spec": spec} if spec else pkg)
        return names

    def _from_setup_py(text: str) -> list[Any]:
        import re as _re
        names: list[Any] = []
        m = _re.search(r"install_requires\s*=\s*\[([\s\S]*?)\]", text, _re.IGNORECASE)
        if m:
            block = m.group(1)
            for q in _re.finditer(r"['\"]([^'\"]+)['\"]", block):
                pkg, spec = _split_name_spec(q.group(1))
                if pkg:
                    names.append({"name": pkg, "version_spec": spec} if spec else pkg)
        return names

    found: list[Any] = []
    for path, b in files.items():
        text = b.decode("utf-8", errors="ignore")
        low = path.lower()
        try:
            if low.endswith("requirements.txt") or low.endswith("requirements-dev.txt") or ("/requirements/" in low and low.endswith(".txt")):
                found.extend(_from_requirements(text))
            elif low == "setup.cfg":
                found.extend(_from_setup_cfg(text))
            elif low == "pyproject.toml":
                found.extend(_from_pyproject(text))
            elif low == "setup.py":
                found.extend(_from_setup_py(text))
        except Exception:
            continue
    # Deduplicate while preserving order
    seen = set()
    dedup: list[Any] = []
    def _key(x: Any) -> str:
        if isinstance(x, dict):
            return (x.get("name") or "").lower()
        return str(x).lower()
    for n in found:
        k = _key(n)
        if not k or k in seen:
            continue
        seen.add(k)
        dedup.append(n)
    logger.info(f"📦 [ANALYZER] Inferred python packages - count={len(dedup)}")
    return dedup


def _infer_env_non_secret_vars(files: Dict[str, bytes]) -> Dict[str, str]:
    """Infer non-secret environment variables and safe defaults from code/config.

    Heuristics:
    - Find usages of os.environ['NAME'], os.environ.get('NAME', default), os.getenv('NAME', default)
    - Exclude likely secrets (names containing KEY|TOKEN|SECRET|PASS|PWD|PRIVATE|AUTH|AWS|GCP|AZURE|DATABASE|POSTGRES|MYSQL|MONGO|REDIS)
    - Prefer explicit code defaults when present; otherwise choose a conservative placeholder
    """
    import re as _re

    names: Dict[str, Optional[str]] = {}

    secret_like = _re.compile(r"(KEY|TOKEN|SECRET|PASS|PWD|PRIVATE|AUTH|AWS|GCP|AZURE|DATABASE|POSTGRES|MYSQL|MONGO|REDIS)", _re.I)

    # Patterns capturing NAME and optional default
    patterns = [
        _re.compile(r"os\.environ\[['\"]([A-Z0-9_]+)['\"]\]"),
        _re.compile(r"os\.environ\.get\(['\"]([A-Z0-9_]+)['\"]\s*,\s*([^\)]+)\)"),
        _re.compile(r"os\.getenv\(['\"]([A-Z0-9_]+)['\"]\s*,\s*([^\)]+)\)"),
    ]

    def _clean_default(expr: str) -> Optional[str]:
        expr = expr.strip()
        # String literal
        if (expr.startswith("'") and expr.endswith("'")) or (expr.startswith('"') and expr.endswith('"')):
            return expr[1:-1]
        # Common bool/int literals
        if expr.lower() in {"true", "false"}:
            return "1" if expr.lower() == "true" else "0"
        if expr.isdigit():
            return expr
        return None

    for path, b in files.items():
        text = b.decode("utf-8", errors="ignore")
        for pat in patterns:
            for m in pat.finditer(text):
                name = m.group(1)
                if not name or secret_like.search(name):
                    continue
                default_val: Optional[str] = None
                if m.lastindex and m.lastindex >= 2:
                    default_val = _clean_default(m.group(2))
                # Keep first-seen default
                names.setdefault(name, default_val)

    def _fallback(name: str) -> Optional[str]:
        upper = name.upper()
        if upper in {"DEBUG", "TESTING"}: return "1"
        if upper in {"ENV", "FLASK_ENV", "FLASK_CONFIG"}: return "development"
        if upper == "SERVER_NAME": return "localhost:5000"
        if upper == "HOST": return "127.0.0.1"
        if upper == "PORT": return "5000"
        return None

    result: Dict[str, str] = {}
    for k, v in names.items():
        chosen = v if v not in (None, "") else _fallback(k)
        if chosen is not None:
            result[k] = str(chosen)

    if result:
        logger.info(f"🌱 [ANALYZER] Inferred env vars - {sorted(result.keys())}")
    return result


def _call_anthropic(prompt: str, max_retries: int = 2, initial_backoff_s: float = 1.0) -> Optional[str]:
    api_key = os.getenv("ANTHROPIC_API_KEY")
    if not api_key or anthropic is None:
        log_llm_operation("API key not available", api_key_set=bool(api_key), anthropic_available=bool(anthropic))
        return None
    
    client = anthropic.Anthropic(api_key=api_key)
    attempt = 0
    backoff = initial_backoff_s
    
    while attempt <= max_retries:
        try:
            log_llm_operation(f"Calling Anthropic API", attempt=attempt + 1, max_retries=max_retries + 1)
            start_time = time.time()
            
            msg = client.messages.create(
                model=os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-20241022"),
                max_tokens=1024,
                temperature=0,
                system=PROMPT_SCHEMA_INSTRUCTIONS,
                messages=[{"role": "user", "content": prompt}],
            )
            
            duration_ms = int((time.time() - start_time) * 1000)
            
            # Extract text
            for block in msg.content:
                if getattr(block, "type", None) == "text":
                    response_text = block.text
                    log_llm_operation("API call successful", duration_ms=duration_ms, 
                                    response_length=len(response_text))
                    return response_text
            
            log_llm_operation("API call returned no text blocks", duration_ms=duration_ms)
            return None
            
        except Exception as e:
            duration_ms = int((time.time() - start_time) * 1000) if 'start_time' in locals() else 0
            log_llm_operation(f"API call failed", attempt=attempt + 1, error=str(e), duration_ms=duration_ms)
            
            if attempt == max_retries:
                return None
            time.sleep(backoff)
            backoff *= 2
            attempt += 1
    return None


def _call_anthropic_messages(messages: list[dict[str, Any]], system: str, max_retries: int = 2, initial_backoff_s: float = 1.0) -> Optional[str]:
    api_key = os.getenv("ANTHROPIC_API_KEY")
    if not api_key or anthropic is None:
        log_llm_operation("API key not available (messages)", api_key_set=bool(api_key), anthropic_available=bool(anthropic))
        return None

    client = anthropic.Anthropic(api_key=api_key)
    attempt = 0
    backoff = initial_backoff_s

    while attempt <= max_retries:
        try:
            log_llm_operation(f"Calling Anthropic API (messages)", attempt=attempt + 1, max_retries=max_retries + 1)
            start_time = time.time()

            msg = client.messages.create(
                model=os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-20241022"),
                max_tokens=1024,
                temperature=0,
                system=system,
                messages=messages,
            )

            duration_ms = int((time.time() - start_time) * 1000)

            for block in msg.content:
                if getattr(block, "type", None) == "text":
                    response_text = block.text
                    log_llm_operation("API call (messages) successful", duration_ms=duration_ms, response_length=len(response_text))
                    return response_text

            log_llm_operation("API call (messages) returned no text blocks", duration_ms=duration_ms)
            return None

        except Exception as e:
            duration_ms = int((time.time() - start_time) * 1000) if 'start_time' in locals() else 0
            log_llm_operation(f"API call (messages) failed", attempt=attempt + 1, error=str(e), duration_ms=duration_ms)

            if attempt == max_retries:
                return None
            time.sleep(backoff)
            backoff *= 2
            attempt += 1
    return None


def _build_prompt(repo_url: str, files: Dict[str, bytes], heur: Dict[str, Any]) -> str:
    """Build comprehensive prompt with prioritized file content"""
    logger.debug(f"📝 [ANALYZER] Building prompt for repo analysis")
    
    # Prepare a compact listing and prioritized excerpts (CI/CD, Docker, tests)
    def _decode(path: str) -> str:
        return files[path][:8000].decode("utf-8", errors="ignore")

    # Priority 1: CI/CD workflows and automation configs
    ci_files = [
        *[p for p in files.keys() if p.startswith(".github/workflows/") and (p.endswith(".yml") or p.endswith(".yaml"))],
        ".circleci/config.yml",
        "Jenkinsfile",
        "azure-pipelines.yml",
        "bitbucket-pipelines.yml",
        ".gitlab-ci.yml",
    ]
    ci_files = [p for p in ci_files if p in files]

    # Priority 2: Container/build descriptors
    build_files = [
        "Dockerfile", "docker-compose.yml", "docker-compose.yaml",
    ]
    build_files = [p for p in build_files if p in files]

    # Priority 3: Standard Python packaging/test configs
    important_files = [
        "pyproject.toml", "poetry.lock", "uv.lock", "Pipfile", "Pipfile.lock",
        "requirements.txt", "requirements/dev.txt", "setup.py", "setup.cfg",
        "tox.ini", "noxfile.py", "Makefile", "README.md", "README.rst",
    ]
    important_files = [p for p in important_files if p in files]

    # Priority 4: Top-level scripts (heuristic)
    top_level_scripts = [
        p for p in files.keys()
        if "/" not in p and (p.endswith(".sh") or p.endswith(".py"))
    ]

    # Priority 5: Scripts referenced by Makefile (best-effort)
    referenced_scripts: list[str] = []
    if "Makefile" in files:
        mk = _decode("Makefile")
        for m in re.findall(r"(?:sh|bash|python)\s+([\./\w\-]+)", mk):
            if m in files and m not in referenced_scripts:
                referenced_scripts.append(m)

    # Build excerpts in priority order with a hard cap
    excerpts = []
    for group_name, group in [
        ("CI/CD", ci_files), 
        ("Build", build_files), 
        ("Config", important_files), 
        ("Referenced", referenced_scripts), 
        ("Scripts", top_level_scripts)
    ]:
        for fname in group:
            excerpts.append({"path": fname, "content": _decode(fname)})
            if len(excerpts) >= 12:  # keep prompt bounded
                break
        if len(excerpts) >= 12:
            break

    file_list = sorted(list(files.keys()))[:300]

    schema_json = json.dumps(ANALYSIS_SCHEMA, indent=2)
    # Include quick inferred python packages to guide the LLM
    inferred_pkgs = _infer_python_packages_from_files(files)
    prompt = (
        f"Repo URL: {repo_url}\n"
        f"Heuristics: {json.dumps(heur)}\n\n"
        f"File list (truncated): {json.dumps(file_list[:100])}\n\n"
        f"Excerpts (truncated): {json.dumps(excerpts[:10])}\n\n"
        f"Inferred python packages (best-effort): {json.dumps(inferred_pkgs[:40])}\n\n"
        f"JSON Schema (strict):\n{schema_json}\n\n"
        f"Return only JSON."
    )
    
    logger.info(f"📝 [ANALYZER] Prompt built - files_included={len(excerpts)}, total_files={len(files)}")
    return prompt


def _build_interactive_context(repo_url: str, files: Dict[str, bytes], heur: Dict[str, Any]) -> Dict[str, Any]:
    """Prepare initial minimal context for interactive LLM analysis."""
    listing = sorted(list(files.keys()))[:500]
    important = [p for p in listing if p in (
        "pyproject.toml", "poetry.lock", "uv.lock", "Pipfile", "Pipfile.lock",
        "requirements.txt", "requirements/dev.txt", "setup.py", "setup.cfg",
        ".github/workflows/ci.yml", ".github/workflows/tests.yml", "Dockerfile",
        "README.md", "README.rst",
    )]
    initial_payload = {
        "repo_url": repo_url,
        "heuristics": heur,
        "known_files": important,
        "dir_candidates": ["requirements", ".github/workflows"],
        "schema": ANALYSIS_SCHEMA,
    }
    return initial_payload


def _extract_json_object(text: str) -> Optional[str]:
    """Best-effort extraction of a JSON object from arbitrary text."""
    logger.debug("🔍 [ANALYZER] Extracting JSON from LLM response")
    
    try:
        # Fast path
        obj = json.loads(text)
        logger.debug("🔍 [ANALYZER] JSON extraction successful (direct parse)")
        return json.dumps(obj)
    except Exception:
        pass
    
    m = re.search(r"\{[\s\S]*\}", text)
    if not m:
        logger.warning("⚠️ [ANALYZER] No JSON object found in text")
        return None
    
    candidate = m.group(0)
    try:
        obj = json.loads(candidate)
        logger.debug("🔍 [ANALYZER] JSON extraction successful (regex extraction)")
        return json.dumps(obj)
    except Exception as e:
        logger.warning(f"⚠️ [ANALYZER] Failed to parse extracted JSON: {e}")
        return None


def _allowlist_commands(cmds: list[str]) -> str:
    """Filter and join commands using allowlist"""
    if not cmds:
        return ""
    
    logger.debug(f"🔒 [ANALYZER] Filtering {len(cmds)} commands through allowlist")
    
    allowed_prefixes = [
        "apt-get install", "apt-get update", "yum install", "dnf install",
        "pip install", "pip3 install", "python -m pip install",
        "poetry install", "poetry add", "uv add", "uv sync", "uv install",
        "npm install", "yarn install", "yarn add",
        "echo ", "export ", "mkdir ", "touch ",
        "python ", "python3 ", "pytest ", "tox ", "nox ",
    ]
    
    safe_cmds = []
    filtered_count = 0
    
    for cmd in cmds:
        cmd = cmd.strip()
        if not cmd:
            continue
        if any(cmd.startswith(prefix) for prefix in allowed_prefixes):
            safe_cmds.append(cmd)
        else:
            logger.warning(f"🔒 [ANALYZER] Filtered unsafe command: {cmd}")
            filtered_count += 1
    
    result = " && ".join(safe_cmds)
    logger.info(f"🔒 [ANALYZER] Command filtering completed - kept={len(safe_cmds)}, filtered={filtered_count}")
    return result


def _filter_system_packages(pkgs: list[str]) -> str:
    """Filter system packages using allowlist"""
    if not pkgs:
        return ""
    
    logger.debug(f"📦 [ANALYZER] Filtering {len(pkgs)} system packages through allowlist")
    
    allowed_packages = {
        "build-essential", "gcc", "g++", "make", "cmake", "ninja-build",
        "git", "curl", "wget", "unzip", "tar", "gzip",
        "python3-dev", "python3-pip", "python-dev", "python-pip",
        "libffi-dev", "libssl-dev", "libbz2-dev", "libreadline-dev",
        "libsqlite3-dev", "libncurses5-dev", "libncursesw5-dev",
        "zlib1g-dev", "liblzma-dev", "tk-dev", "uuid-dev",
        "pkg-config", "autoconf", "automake", "libtool",
        "nodejs", "npm", "yarn", "redis-server", "postgresql-dev",
    }
    
    safe_pkgs = []
    filtered_count = 0
    
    for pkg in pkgs:
        pkg = pkg.strip()
        if pkg in allowed_packages:
            safe_pkgs.append(pkg)
        else:
            logger.warning(f"📦 [ANALYZER] Filtered unsafe package: {pkg}")
            filtered_count += 1
    
    result = " ".join(safe_pkgs)
    logger.info(f"📦 [ANALYZER] Package filtering completed - kept={len(safe_pkgs)}, filtered={filtered_count}")
    return result


def submit_analysis(repo_url: str) -> str:
    """Submit analysis job for a repository"""
    job_id = f"{_slug(repo_url)}-analysis"
    
    logger.info(f"🚀 [ANALYZER] Submitting analysis job - repo={repo_url}, job_id={job_id}")
    
    with _jobs_lock:
        jobs = _load_jobs()
        jobs[job_id] = AnalysisJob(
            job_id=job_id,
            repo_url=repo_url,
            status="pending",
            message="analysis queued",
        )
        _save_jobs(jobs)
    
    # Submit to thread pool
    future = _executor.submit(_run_analysis, job_id, repo_url)
    logger.debug(f"🔄 [ANALYZER] Analysis job submitted to thread pool - job_id={job_id}")
    
    return job_id


def job_status(job_id: str) -> AnalysisJob:
    """Get status of analysis job"""
    with _jobs_lock:
        jobs = _load_jobs()
        if job_id not in jobs:
            logger.warning(f"⚠️ [ANALYZER] Job status requested for unknown job_id: {job_id}")
            raise KeyError(f"Analysis job {job_id} not found")
        
        job = jobs[job_id]
        logger.debug(f"📊 [ANALYZER] Job status requested - job_id={job_id}, status={job.status}")
        return job


def job_result(job_id: str) -> Optional[Dict[str, Any]]:
    """Get result of completed analysis job"""
    with _jobs_lock:
        jobs = _load_jobs()
        if job_id in jobs and jobs[job_id].status == "succeeded":
            result = jobs[job_id].result
            logger.debug(f"📋 [ANALYZER] Job result requested - job_id={job_id}, has_result={bool(result)}")
            return result
        
        logger.debug(f"📋 [ANALYZER] Job result requested but not ready - job_id={job_id}")
        return None


def _run_analysis(job_id: str, repo_url: str) -> None:
    start_time = time.time()
    log_analysis_start(repo_url, job_id)
    
    with _jobs_lock:
        jobs = _load_jobs()
        if job_id in jobs:
            jobs[job_id].status = "running"
            _save_jobs(jobs)
    
    try:
        # Fetch repository tarball
        log_analysis_step("Fetching repository tarball", job_id, repo_url=repo_url)
        tar_url = _github_tarball_url(repo_url)
        headers = {}
        gh_token = os.getenv("GITHUB_TOKEN")
        if gh_token:
            headers["Authorization"] = f"token {gh_token}"
            logger.debug(f"🔑 [ANALYZER] Using GitHub token for authentication")
        
        try:
            r = requests.get(tar_url, headers=headers, timeout=60)
            r.raise_for_status()
            log_analysis_step("Tarball fetched successfully", job_id, 
                            size_bytes=len(r.content), status_code=r.status_code)
        except requests.HTTPError as he:  # type: ignore[attr-defined]
            code = getattr(he.response, "status_code", None)
            reason = getattr(he.response, "reason", "")
            msg = f"GitHub tarball fetch failed: HTTP {code} {reason}" if code else "GitHub tarball fetch failed"
            if code == 404:
                msg = "Repository not found or inaccessible (HTTP 404). If private, set GITHUB_TOKEN."
            elif code in (401, 403):
                msg = f"Unauthorized or access denied (HTTP {code}). Check GITHUB_TOKEN permissions."
            
            duration_ms = int((time.time() - start_time) * 1000)
            log_analysis_error(repo_url, job_id, msg, duration_ms)
            
            with _jobs_lock:
                jobs = _load_jobs()
                if job_id in jobs:
                    jobs[job_id].status = "failed"
                    jobs[job_id].message = msg
                    _save_jobs(jobs)
            return
        except (requests.ConnectionError, requests.Timeout):  # type: ignore[attr-defined]
            msg = "Network error while fetching repository tarball (connection/timeout)."
            duration_ms = int((time.time() - start_time) * 1000)
            log_analysis_error(repo_url, job_id, msg, duration_ms)
            
            with _jobs_lock:
                jobs = _load_jobs()
                if job_id in jobs:
                    jobs[job_id].status = "failed"
                    jobs[job_id].message = msg
                    _save_jobs(jobs)
            return
        
        # Extract files and run heuristics
        log_analysis_step("Extracting files from tarball", job_id)
        files = _list_files_from_tarball(r.content)
        
        log_analysis_step("Running heuristics analysis", job_id, file_count=len(files))
        heur = _heuristics(files)
        
        log_analysis_step("Building LLM prompt", job_id)
        prompt = _build_prompt(repo_url, files, heur)
        
        # Try LLM with retries and robust parsing; fallback to heuristics if needed
        result: Optional[Dict[str, Any]] = None
        llm_used = False
        llm_attempts = 3
        last_parse_error: Optional[str] = None
        
        log_analysis_step("Attempting LLM analysis", job_id, max_attempts=llm_attempts)

        # Interactive short loop (<=5 turns) to request additional metadata
        try:
            interactive_turns = 0
            context = _build_interactive_context(repo_url, files, heur)
            convo: list[Dict[str, Any]] = [
                {"role": "user", "content": json.dumps(context)},
            ]
            while interactive_turns < 5:
                interactive_turns += 1
                raw = _call_anthropic_messages(convo, INTERACTIVE_INSTRUCTIONS)
                if not raw:
                    break
                data = _extract_json_object(raw)
                if not data:
                    break
                try:
                    msg = json.loads(data)
                except Exception:
                    break
                if isinstance(msg, dict) and msg.get("type") == "final" and msg.get("analysis"):
                    candidate = msg["analysis"]
                    validate(instance=candidate, schema=ANALYSIS_SCHEMA)
                    result = candidate
                    llm_used = True
                    log_analysis_step("Interactive LLM produced final result", job_id, turns=interactive_turns)
                    break
                # Handle requests
                if isinstance(msg, dict) and msg.get("type") == "request":
                    action = msg.get("action")
                    if action == "need_files":
                        req_paths = [p for p in msg.get("paths", []) if isinstance(p, str)]
                        provided: Dict[str, str] = {}
                        for p in req_paths:
                            # Restrict to allowlist of metadata patterns
                            if not any(fnmatch(p, pat) for pat in [
                                "pyproject.toml", "poetry.lock", "uv.lock", "Pipfile", "Pipfile.lock",
                                "requirements*.txt", "requirements/*.txt",
                                "setup.py", "setup.cfg",
                                "README.*", "docs/requirements*.txt",
                                "Dockerfile", "docker-compose.*",
                                ".github/workflows/*.yml", ".github/workflows/*.yaml",
                            ]):
                                continue
                            if p in files:
                                provided[p] = files[p][:8000].decode("utf-8", errors="ignore")
                        context["provided_files"] = provided
                    elif action == "need_dirs":
                        req_dirs = [d for d in msg.get("dirs", []) if isinstance(d, str)]
                        listings: Dict[str, list[str]] = {}
                        for d in req_dirs:
                            # Only known metadata directories
                            if d not in {"requirements", ".github/workflows", "docs"}:
                                continue
                            listings[d] = sorted([p for p in files.keys() if p.startswith(d + "/")])[:100]
                        context["provided_dirs"] = listings
                    else:
                        break
                    # Continue loop to allow at most 5 turns; append assistant reply and a new user turn
                    convo.append({"role": "assistant", "content": json.dumps(msg)})
                    convo.append({"role": "user", "content": json.dumps(context)})
                else:
                    break
        except Exception:
            pass
        
        for attempt in range(llm_attempts):
            log_analysis_step(f"LLM attempt {attempt + 1}", job_id)
            raw = _call_anthropic(prompt)
            if raw is None:
                log_analysis_step("LLM call returned None", job_id, attempt=attempt + 1)
                break
            
            extracted = _extract_json_object(raw)
            if not extracted:
                last_parse_error = "invalid or missing JSON object in LLM output"
                log_analysis_step("JSON extraction failed", job_id, attempt=attempt + 1, error=last_parse_error)
                continue
            
            try:
                candidate = json.loads(extracted)
                log_analysis_step("JSON parsing successful", job_id, attempt=attempt + 1)
            except Exception as e:
                last_parse_error = f"json parse error: {e}"
                log_analysis_step("JSON parsing failed", job_id, attempt=attempt + 1, error=last_parse_error)
                continue
            
            # Validate against schema
            try:
                validate(instance=candidate, schema=ANALYSIS_SCHEMA)
                result = candidate
                llm_used = True
                log_analysis_step("Schema validation successful", job_id, attempt=attempt + 1)
                break
            except ValidationError as ve:
                last_parse_error = f"schema validation failed: {ve.message}"
                log_analysis_step("Schema validation failed", job_id, attempt=attempt + 1, error=last_parse_error)
                continue

        if result is None:
            # Fallback to heuristics-only minimal result
            log_analysis_step("Falling back to heuristics-only result", job_id, 
                            reason=last_parse_error or "LLM unavailable")
            result = {
                "schema_version": "1.0",
                "repo_url": repo_url,
                "package_manager": heur.get("package_manager", "unknown"),
                "python_version": "unknown",
                "python_packages": _infer_python_packages_from_files(files),
                "system_packages": [],
                "install": {
                    "pre_install_cmds": [],
                    "install_cmds": [],
                    "post_install_cmds": [],
                },
                "codeflash": {
                    "module_root": heur.get("module_root", "auto"),
                    "tests_root": heur.get("tests_root", "auto"),
                    "formatter_cmds": ["disabled"],
                },
                "tests": {"test_command": "pytest", "test_framework": "pytest"},
                "resources": {"tier": "small", "rationale": "heuristic default"},
                "env": {"non_secret_env_vars": {}},
                "confidence": 0.4,
                "evidence": list(files.keys())[:20],
            }

        # Post-process: allowlist commands and packages
        log_analysis_step("Post-processing analysis result", job_id)
        result_install = result.get("install", {})
        pre = _allowlist_commands(result_install.get("pre_install_cmds", []))
        ins = _allowlist_commands(result_install.get("install_cmds", []))
        post = _allowlist_commands(result_install.get("post_install_cmds", []))
        result_install["pre_install_cmds"] = pre.split(" && ") if pre else []
        result_install["install_cmds"] = ins.split(" && ") if ins else []
        result_install["post_install_cmds"] = post.split(" && ") if post else []
        result["install"] = result_install

        sys_pkgs = _filter_system_packages(result.get("system_packages", []))
        result["system_packages"] = sys_pkgs.split() if sys_pkgs else []

        # Normalize python_packages list (unique, non-empty, parse objects)
        py_pkgs_raw = result.get("python_packages", []) or []
        normalized: list[Any] = []
        seen_pk: set[str] = set()
        if isinstance(py_pkgs_raw, list):
            for item in py_pkgs_raw:
                pkg_name: Optional[str] = None
                version_spec: Optional[str] = None
                if isinstance(item, dict):
                    pkg_name = str(item.get("name") or "").strip() or None
                    version_spec = str(item.get("version_spec") or "").strip() or None
                elif isinstance(item, str):
                    s = item.strip()
                    # Try to parse stringified dicts like "{'name': 'flask', 'version_spec': '>=1.0'}"
                    if s.startswith("{") and s.endswith("}"):
                        try:
                            parsed = ast.literal_eval(s)
                            if isinstance(parsed, dict):
                                pkg_name = str(parsed.get("name") or "").strip() or None
                                version_spec = str(parsed.get("version_spec") or "").strip() or None
                        except Exception:
                            pass
                    if pkg_name is None:
                        # Treat as pip spec; split on operators to find name
                        for sep in ["===", "==", ">=", "<=", "!=", "~=", ">", "<", "^", "~"]:
                            if sep in s:
                                parts = s.split(sep, 1)
                                pkg_name = parts[0].strip() or None
                                version_spec = sep + parts[1].strip()
                                break
                        if pkg_name is None:
                            pkg_name = s or None
                if not pkg_name:
                    continue
                key = pkg_name.lower()
                if key in seen_pk:
                    continue
                seen_pk.add(key)
                normalized.append({"name": pkg_name, "version_spec": version_spec} if version_spec else {"name": pkg_name})
        result["python_packages"] = normalized

        # Derive test_framework from test_command if not already set
        try:
            tests_block = result.get("tests", {}) or {}
            test_command = tests_block.get("test_command", "")
            test_framework = tests_block.get("test_framework")
            
            if not test_framework and test_command:
                # Derive test framework from test command (same logic as run_optimization.sh)
                test_cmd_lower = test_command.lower()
                if "pytest" in test_cmd_lower:
                    test_framework = "pytest"
                elif "unittest" in test_cmd_lower:
                    test_framework = "unittest"
                elif "nose" in test_cmd_lower or "nosetests" in test_cmd_lower:
                    test_framework = "nose"
                else:
                    test_framework = "pytest"  # default fallback
                
                tests_block["test_framework"] = test_framework
                result["tests"] = tests_block
                log_analysis_step("Derived test framework from command", job_id, 
                                test_command=test_command, test_framework=test_framework)
        except Exception as e:
            log_analysis_step("Failed to derive test framework", job_id, error=str(e))

        # Infer additional env vars (non-secret) and merge
        try:
            inferred_env = _infer_env_non_secret_vars(files)
            env_block = result.get("env", {}) or {}
            cur = env_block.get("non_secret_env_vars", {}) or {}
            if isinstance(cur, dict):
                merged = {**inferred_env, **{k: v for k, v in cur.items() if isinstance(k, str)}}
            else:
                merged = inferred_env
            env_block["non_secret_env_vars"] = merged
            result["env"] = env_block
        except Exception:
            pass

        # Persist to file
        log_analysis_step("Persisting analysis result", job_id)
        out_path = ANALYSIS_DIR / f"{_slug(repo_url)}.json"
        out_path.write_text(json.dumps(result, indent=2), encoding="utf-8")

        duration_ms = int((time.time() - start_time) * 1000)
        
        with _jobs_lock:
            jobs = _load_jobs()
            if job_id in jobs:
                jobs[job_id].status = "succeeded"
                if llm_used:
                    jobs[job_id].message = "analysis completed via LLM"
                else:
                    reason = last_parse_error or ("LLM unavailable" if os.getenv("ANTHROPIC_API_KEY") else "ANTHROPIC_API_KEY not set")
                    jobs[job_id].message = f"analysis completed via heuristics ({reason})"
                jobs[job_id].result = result
                _save_jobs(jobs)
        
        log_analysis_success(repo_url, job_id, duration_ms, 
                           method="LLM" if llm_used else "heuristics",
                           confidence=result.get("confidence", 0),
                           package_manager=result.get("package_manager", "unknown"))
        
    except Exception as e:
        duration_ms = int((time.time() - start_time) * 1000)
        log_analysis_error(repo_url, job_id, str(e), duration_ms)
        
        with _jobs_lock:
            jobs = _load_jobs()
            if job_id in jobs:
                jobs[job_id].status = "failed"
                jobs[job_id].message = str(e)
                _save_jobs(jobs)


def load_analysis_for_repo(repo_url: str) -> Optional[Dict[str, Any]]:
    """Load saved analysis result for a repository"""
    path = ANALYSIS_DIR / f"{_slug(repo_url)}.json"
    
    if not path.exists():
        logger.debug(f"📂 [ANALYZER] No analysis file found for repo: {repo_url}")
        return None
    
    try:
        result = json.loads(path.read_text(encoding="utf-8"))
        logger.debug(f"📂 [ANALYZER] Loaded analysis result for repo: {repo_url}")
        return result
    except Exception as e:
        logger.warning(f"⚠️ [ANALYZER] Failed to load analysis file for {repo_url}: {e}")
        return None