codeflash-internal/tiles/codeflash-internal-skills/evals/summary_infeasible.json

{
  "package_name": "codeflash-internal-skills",
  "total_infeasible": 3,
  "infeasible_capabilities": [
    {
      "capability": "End-to-end LLM call with real API keys",
      "reason": "Testing actual LLM calls (Step 4 of debug-optimization-failure, Step 5 of debug-test-generation) requires live API keys for Azure OpenAI and Anthropic Foundry. Eval environments cannot safely provision these credentials, and mocking would defeat the purpose of testing the real integration.",
      "skills_affected": ["debug-optimization-failure", "debug-test-generation"]
    },
    {
      "capability": "PostHog event tracking verification",
      "reason": "Step 7 of debug-optimization-failure references PostHog events (aiservice-optimize-openai-usage). Verifying that events are sent correctly requires a PostHog instance and API access, which is not available in eval environments.",
      "skills_affected": ["debug-optimization-failure"]
    },
    {
      "capability": "Sentry error capture verification",
      "reason": "Step 6 of debug-optimization-failure references Sentry message capture when candidates fail to parse. Verifying Sentry integration requires a live Sentry DSN and cannot be tested in isolation without the full Django runtime.",
      "skills_affected": ["debug-optimization-failure"]
    }
  ]
}