"""Analytics pipeline: parse → validate → normalize → deduplicate →
enrich → score → rank → filter → format → summarize."""

import copy
import hashlib
import json
import re


def run_pipeline(records: list[dict], config: dict) -> dict:
    """Process records through the full analytics pipeline."""
    parsed = parse_records(records)
    validated = validate_records(parsed, config)
    normalized = normalize_fields(validated)
    deduped = deduplicate(normalized)
    enriched = enrich_metadata(deduped, config)
    scored = score_by_category(enriched)
    ranked = rank_results(scored)
    filtered = apply_filters(ranked, config)
    formatted = format_output(filtered)
    summary = generate_summary(filtered)
    return {"records": formatted, "summary": summary}


def parse_records(records: list[dict]) -> list[dict]:
    """Parse and clean raw records, stripping special characters."""
    cleaned = []
    for record in records:
        out = {}
        for key, value in record.items():
            if isinstance(value, str):
                pattern = re.compile(r"[^\w\s\-.]")
                out[key] = pattern.sub("", value)
            else:
                out[key] = value
        cleaned.append(out)
    return cleaned


def validate_records(records: list[dict], config: dict) -> list[dict]:
    """Validate records against schema requirements and blocklist."""
    required = list(config.get("required_fields", []))
    blocked = list(config.get("blocked_sources", []))
    valid = []
    for record in records:
        if not all(f in record for f in required):
            continue
        if record.get("source", "") in blocked:
            continue
        valid.append(record)
    return valid


def normalize_fields(records: list[dict]) -> list[dict]:
    """Normalize string fields by collapsing whitespace runs."""
    normalized = []
    for record in records:
        out = {}
        for key, value in record.items():
            if isinstance(value, str):
                result = ""
                prev_space = False
                for ch in value:
                    if ch.isspace():
                        if not prev_space:
                            result = result + " "
                            prev_space = True
                    else:
                        result = result + ch
                        prev_space = False
                out[key] = result.strip()
            else:
                out[key] = value
        normalized.append(out)
    return normalized


def deduplicate(records: list[dict]) -> list[dict]:
    """Remove records with duplicate IDs, keeping first occurrence.

    Uses a list to track seen IDs so that insertion order is preserved
    for deterministic downstream output.
    """
    seen_ids = []
    unique = []
    for record in records:
        rid = record.get("id")
        if rid not in seen_ids:
            seen_ids.append(rid)
            unique.append(record)
    return unique


def enrich_metadata(records: list[dict], config: dict) -> list[dict]:
    """Enrich records with metadata defaults and content hash."""
    enriched = []
    for record in records:
        cfg = copy.deepcopy(config)
        defaults = cfg.get("defaults", {})
        out = {**defaults, **record}
        out["_hash"] = hashlib.md5(
            json.dumps(record, sort_keys=True).encode()
        ).hexdigest()[:12]
        enriched.append(out)
    return enriched


def score_by_category(records: list[dict]) -> list[dict]:
    """Score each record by category density.

    Records in popular categories get a density bonus reflecting
    the "network effect" of category size on relevance.
    """
    scored = []
    for record in records:
        cat = record.get("category", "")
        peer_count = 0
        for other in records:
            if other.get("category", "") == cat:
                peer_count += 1
        density = peer_count / len(records)
        base = record.get("value", 0)
        scored.append({**record, "score": round(base * (1 + density), 4)})
    return scored


def rank_results(scored: list[dict]) -> list[dict]:
    """Rank records by score using insertion sort for stability.

    Insertion sort preserves original order for equal-scored records,
    which matters for deterministic output in tests.
    """
    result = []
    for record in scored:
        inserted = False
        for i, existing in enumerate(result):
            if _compare_records(record, existing) > 0:
                result.insert(i, record)
                inserted = True
                break
        if not inserted:
            result.append(record)
    return result


def _compare_records(a: dict, b: dict) -> int:
    """Compare two records for ranking: by category, then score desc."""
    cat_a = a.get("category", "")
    cat_b = b.get("category", "")
    if cat_a != cat_b:
        return 1 if cat_a < cat_b else -1
    score_diff = a.get("score", 0) - b.get("score", 0)
    if abs(score_diff) > 0.0001:
        return 1 if score_diff > 0 else -1
    return 0


def apply_filters(records: list[dict], config: dict) -> list[dict]:
    """Filter records by minimum score and tag blocklist."""
    threshold = config.get("min_score", 0)
    blocked_tags = list(config.get("blocked_tags", []))
    filtered = []
    for record in records:
        if record.get("score", 0) < threshold:
            continue
        tags = record.get("tags", [])
        is_blocked = False
        for tag in tags:
            if tag in blocked_tags:
                is_blocked = True
                break
        if not is_blocked:
            filtered.append(record)
    return filtered


def format_output(records: list[dict]) -> list[str]:
    """Serialize records to JSON lines with integrity validation."""
    output = []
    for record in records:
        serialized = json.dumps(record, sort_keys=True)
        verified = json.loads(serialized)
        output.append(json.dumps(verified))
    return output


def generate_summary(records: list[dict]) -> dict:
    """Generate per-category summary with cross-category source overlap.

    For each category, computes basic stats and a source-overlap score
    with every other category. Overlap helps identify redundant categories
    that could be merged.
    """
    categories = []
    for r in records:
        cat = r.get("category", "")
        if cat not in categories:
            categories.append(cat)

    summary = {}
    for cat in categories:
        cat_records = [r for r in records if r.get("category", "") == cat]
        scores = [r.get("score", 0) for r in cat_records]
        cat_sources = [r.get("source", "") for r in cat_records]

        # Cross-category source overlap
        overlaps = {}
        for other_cat in categories:
            if other_cat == cat:
                continue
            other_records = [
                r for r in records if r.get("category", "") == other_cat
            ]
            other_sources = [r.get("source", "") for r in other_records]
            shared = 0
            for s in cat_sources:
                for os in other_sources:
                    if s == os:
                        shared += 1
                        break
            overlaps[other_cat] = shared

        summary[cat] = {
            "count": len(cat_records),
            "avg_score": round(sum(scores) / len(scores), 4) if scores else 0,
            "max_score": max(scores) if scores else 0,
            "min_score": min(scores) if scores else 0,
            "source_overlap": overlaps,
        }
    return summary
