codeflash-internal/tiles/codeflash-internal-docs/evals/scenario-2/criteria.json
2026-02-14 22:25:30 -05:00

31 lines
1.6 KiB
JSON

{
"context": "Tests whether the agent understands the language dispatch pattern, endpoint registration conventions, and Django-Ninja schema patterns from the aiservice endpoints and optimization pipeline docs.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Language dispatch pattern",
"description": "The router dispatches by data.language with explicit checks for 'javascript'/'typescript', 'java', and a default for Python. All three branches are present.",
"max_score": 25
},
{
"name": "Lazy imports",
"description": "Language-specific handler imports are inside the function body (not at module top level) to avoid circular dependencies, matching the existing optimize_api and testgen_api pattern.",
"max_score": 20
},
{
"name": "Async endpoint with auth",
"description": "The endpoint handler is declared as async def and uses AuthenticatedRequest (HttpBearer + django_auth) for authentication. Response schema uses status code mapping: {200: SuccessSchema, 400: ErrorSchema, 500: ErrorSchema}.",
"max_score": 25
},
{
"name": "Schema follows conventions",
"description": "CodeReviewSchema inherits from ninja.Schema, includes source_code (str), trace_id (str), language (str, default 'python'), and any additional fields use proper typing.",
"max_score": 15
},
{
"name": "URL registration",
"description": "Shows registration in urls.py using api.add_router or urlpatterns with the path '/ai/code_review' and the code_review_api instance, consistent with existing endpoint registrations.",
"max_score": 15
}
]
}