from processor.core import process_records


def test_process_basic():
    config = {
        "required_fields": ["id", "value"],
        "defaults": {"status": "pending", "priority": 0},
    }
    records = [
        {"id": 1, "value": "a", "source": "web"},
        {"id": 2, "value": "b", "source": "api"},
        {"id": 1, "value": "a", "source": "web"},  # duplicate
        {"value": "c"},  # missing id, should be filtered
    ]
    result = process_records(records, config)
    assert len(result) == 2  # 2 unique valid records


def test_process_large_batch():
    """Production batch — process_records is too slow and uses too much memory."""
    config = {
        "required_fields": ["id", "value", "source"],
        "defaults": {
            "status": "pending",
            "priority": 0,
            "metadata": {
                "pipeline": "v3",
                "region": "us-east",
                "settings": {
                    "retry_count": 3,
                    "backoff": [1, 2, 4, 8, 16],
                    "timeout": {"connect": 5, "read": 30},
                },
            },
            "tags": ["ingested", "validated"],
            "audit": {
                "created_by": "pipeline-worker",
                "reviewers": ["system", "auto-qa"],
                "checks": {
                    "schema": True,
                    "dedup": True,
                    "enrichment": False,
                },
            },
            "routing": {
                "queue": "default",
                "priority_levels": [1, 2, 3, 4, 5],
                "retry_policy": {
                    "max_retries": 5,
                    "delay_ms": [100, 500, 2000],
                },
            },
        },
    }
    sources = [f"source-{i}" for i in range(20)]
    records = []
    for i in range(5_000):
        base_id = i % 4_000  # ~20% duplicates
        records.append(
            {
                "id": base_id,
                "value": f"data-{base_id}",
                "source": sources[base_id % len(sources)],
                "timestamp": f"2024-01-{(base_id % 28) + 1:02d}",
            }
        )
    result = process_records(records, config)
    assert len(result) == 4_000
