{
  "name": "ranking",
  "description": "4 pipeline functions with 1 hot bottleneck (97.6%) and 3 cold antipatterns. Tests profiling, prioritization, and thoroughness.",
  "eval_type": "ranking",
  "test_command": "PYTHONPATH=src uv run python -m pytest tests/ -v",
  "bugs": [
    {
      "id": "score-on2",
      "file": "src/pipeline/core.py",
      "function": "score_records",
      "domain": "data-structures",
      "description": "O(n²) nested loop counting same-category peers",
      "expected_fix": "Pre-compute category counts with Counter/defaultdict",
      "impact_pct": 97.6
    },
    {
      "id": "clean-string-concat",
      "file": "src/pipeline/core.py",
      "function": "clean_records",
      "domain": "data-structures",
      "description": "String concatenation in loop (result = result + ch) — O(n²) per string",
      "expected_fix": "Use list append + join, or regex",
      "impact_pct": 1.7
    },
    {
      "id": "format-json-roundtrip",
      "file": "src/pipeline/core.py",
      "function": "format_records",
      "domain": "data-structures",
      "description": "JSON serialize + deserialize per record for type normalization",
      "expected_fix": "Remove if types are already clean; or batch serialize",
      "impact_pct": 0.5
    },
    {
      "id": "validate-list-blocklist",
      "file": "src/pipeline/core.py",
      "function": "validate_records",
      "domain": "data-structures",
      "description": "Blocklist stored as list — O(n) lookup per record",
      "expected_fix": "Convert blocklist to set",
      "impact_pct": 0.2
    }
  ],
  "test_file": "tests/test_pipeline.py",
  "test_name": "test_large_batch",
  "data_size": 5000,
  "rubric": {
    "criteria": {
      "profiled_and_identified": 3,
      "fixed_all_actionable_targets": 5,
      "tests_pass": 2,
      "ran_adversarial_review": 1
    },
    "total": 11,
    "notes": {
      "profiled_and_identified": "Used a profiler (cProfile, tracemalloc, or similar) and identified the performance bottlenecks with evidence. Must show actual profiling output, not just source-level guesses. Full credit for profiling with impact quantification.",
      "fixed_all_actionable_targets": "Fixed ALL targets that showed measurable impact — not just the dominant one. Full credit (5) for fixing all 4 bugs. 3-4 points for fixing 3. 1-2 points for fixing 2. Zero if only fixed 1. Order does not matter.",
      "tests_pass": "All tests pass after optimization and the improvement is verified with before/after measurement.",
      "ran_adversarial_review": "Ran a Codex adversarial review (codex-companion.mjs adversarial-review) before declaring completion. Full credit if the review was invoked and its output was acknowledged."
    }
  }
}
