{ "updatedAt": "2026-07-21T03:17:51.036Z", "source": "openrouter-models-api", "rankingVersion": "2026-07-21.v1", "probeMode": "active", "refreshMode": "fullEval", "liteEvalMode": "budget_exhausted", "liteEvalSuite": "lite-agent-eval-v1", "rankingConfidence": "low", "rankingConfidenceReason": "The lite eval stage stopped before all threatening candidates were resolved.", "evalCoverage": { "evaluated": 8, "completedToday": 6, "attemptedToday": 8, "totalRanked": 15, "candidatePool": 15, "unevaluatedCanStillWin": true, "unevaluatedThreatCount": 3, "topUnevaluatedThreat": "nvidia/nemotron-nano-9b-v2:free", "maxUnevaluatedPossibleScore": 1329 }, "baseUrl": "https://openrouter.ai/api/v1", "createKeyUrl": "https://openrouter.ai/settings/keys", "fallback": { "id": "openrouter/free", "reason": "OpenRouter-managed free model router" }, "count": 6, "models": [ { "rank": 1, "id": "cohere/north-mini-code:free", "name": "Cohere: North Mini Code (free)", "score": 1289, "metadataScore": 514, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 650, "contextLength": 256000, "maxCompletionTokens": 64000, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": true, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 2, "total": 3, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "find . -name \"*.log\" -exec grep -l \"FATAL:\" {} \\;", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ] }, "reason": "Tools, reasoning, 128k+ context; lite eval 2/3", "instabilityPenalty": 0 }, { "rank": 2, "id": "poolside/laguna-xs-2.1:free", "name": "Poolside: Laguna XS 2.1 (free)", "score": 1135, "metadataScore": 490, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 520, "contextLength": 262144, "maxCompletionTokens": 32768, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": false, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 1, "total": 3, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 220, "maxPoints": 350, "score": 0.629, "status": "partial", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": false, "readmeHasTitle": false, "gitignoreCreated": false, "gitignoreHasPycache": false, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "find . -name \"*.log\" -exec grep -l \"FATAL:\" {} + | sort -u", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ] }, "reason": "Tools, reasoning, 262k context; lite eval 1/3", "instabilityPenalty": 0 }, { "rank": 3, "id": "poolside/laguna-m.1:free", "name": "Poolside: Laguna M.1 (free)", "score": 1120, "metadataScore": 475, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 520, "contextLength": 262144, "maxCompletionTokens": 32768, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": false, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 1, "total": 3, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 220, "maxPoints": 350, "score": 0.629, "status": "partial", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": false, "readmeHasTitle": false, "gitignoreCreated": false, "gitignoreHasPycache": false, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "find . -name \"*.log\" -exec grep -l \"FATAL:\" {} +", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ] }, "reason": "Tools, reasoning, 262k context; lite eval 1/3", "instabilityPenalty": 0 }, { "rank": 4, "id": "nvidia/nemotron-3-ultra-550b-a55b:free", "name": "NVIDIA: Nemotron 3 Ultra (free)", "score": 920, "metadataScore": 495, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 300, "contextLength": 1000000, "maxCompletionTokens": 65536, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 1, "total": 3, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 0, "maxPoints": 350, "score": 0, "status": "failed", "details": { "usedTool": false, "mainPyCreated": false, "helloPython": false, "readmeCreated": false, "readmeHasTitle": false, "gitignoreCreated": false, "gitignoreHasPycache": false, "noUnsafePaths": false } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "grep -r -l \"FATAL:\" --include=\"*.log\" .", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "The user wants me to decode only the time after the em dash (—) in a Babylonian-" } } ] }, "reason": "Tools, reasoning, 262k context; lite eval 1/3", "instabilityPenalty": 0 }, { "rank": 5, "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", "name": "NVIDIA: Nemotron 3 Nano Omni (free)", "score": 919, "metadataScore": 494, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 300, "contextLength": 256000, "maxCompletionTokens": 65536, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 1, "total": 3, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 0, "maxPoints": 350, "score": 0, "status": "failed", "details": { "usedTool": false, "mainPyCreated": false, "helloPython": false, "readmeCreated": false, "readmeHasTitle": false, "gitignoreCreated": false, "gitignoreHasPycache": false, "noUnsafePaths": false } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "find . -name \"*.log\" -exec grep -l \"FATAL:\" {} \\;", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ] }, "reason": "Tools, reasoning, 128k+ context; lite eval 1/3", "instabilityPenalty": 0 }, { "rank": 6, "id": "google/gemma-4-26b-a4b-it:free", "name": "Google: Gemma 4 26B A4B (free)", "score": 880, "metadataScore": 540, "healthScore": 400, "latencyScore": 40, "liteEvalScore": 0, "contextLength": 131072, "maxCompletionTokens": 32768, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": true, "supportsResponseFormat": true, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": true, "latencyMs": 2221, "healthStatus": "passed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "model_rate_limited", "passed": 0, "total": 1, "updatedAt": "2026-07-21T03:17:51.036Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 0, "maxPoints": 350, "score": 0, "status": "model_rate_limited", "details": { "responseStatus": 429, "rateLimitKind": "model_or_provider_limit", "providerName": "Google AI Studio", "rateLimitLimit": null, "rateLimitRemaining": null, "rateLimitResetAt": null, "responsePreview": "{\"error\":{\"message\":\"Provider returned error\",\"code\":429,\"metadata\":{\"raw\":\"google/gemma-4-26b-a4b-it:free is temporarily rate-limited upstream. Please retry sh" } } ] }, "reason": "Tools, structured outputs, reasoning", "instabilityPenalty": 100 } ], "notes": [ "Free model availability and rate limits can vary.", "Lite agent eval did not complete during this refresh, so practical-task scores may be missing or cached.", "Use openrouter/free if the recommended model is temporarily unavailable." ] }