| task_id,maka,codex,claude-code,reasonix,maka_failure_class,codex_failure_class,claude-code_failure_class,reasonix_failure_class |
| adaptive-rejection-sampler,passed,passed,failed,unscored,,,budget_exhausted,infra_failed |
| bn-fit-modify,passed,passed,failed,passed,,,verification_failed, |
| break-filter-js-from-html,passed,passed,passed,passed,,,, |
| build-cython-ext,passed,passed,passed,passed,,,, |
| build-pmars,passed,passed,passed,passed,,,, |
| build-pov-ray,passed,passed,passed,passed,,,, |
| caffe-cifar-10,passed,passed,passed,passed,,,, |
| cancel-async-tasks,failed,passed,failed,passed,verification_failed,,budget_exhausted, |
| chess-best-move,failed,failed,failed,failed,verification_failed,budget_exhausted,budget_exhausted,budget_exhausted |
| circuit-fibsqrt,passed,passed,passed,failed,,,,verification_failed |
| cobol-modernization,passed,passed,passed,passed,,,, |
| code-from-image,failed,passed,failed,passed,budget_exhausted,,budget_exhausted, |
| compile-compcert,passed,passed,passed,passed,,,, |
| configure-git-webserver,failed,passed,passed,passed,verification_failed,,, |
| constraints-scheduling,passed,passed,passed,passed,,,, |
| count-dataset-tokens,passed,passed,passed,failed,,,,verification_failed |
| crack-7z-hash,passed,passed,passed,failed,,,,budget_exhausted |
| custom-memory-heap-crash,passed,passed,passed,passed,,,, |
| db-wal-recovery,passed,passed,failed,passed,,,budget_exhausted, |
| distribution-search,passed,passed,passed,passed,,,, |
| dna-assembly,failed,passed,failed,failed,verification_failed,,budget_exhausted,verification_failed |
| dna-insert,failed,passed,passed,failed,verification_failed,,,verification_failed |
| extract-elf,passed,failed,failed,failed,,verification_failed,verification_failed,verification_failed |
| extract-moves-from-video,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| feal-differential-cryptanalysis,passed,failed,passed,passed,,verification_failed,, |
| feal-linear-cryptanalysis,passed,passed,passed,passed,,,, |
| filter-js-from-html,failed,failed,failed,failed,verification_failed,verification_failed,verification_failed,verification_failed |
| financial-document-processor,passed,passed,passed,passed,,,, |
| fix-code-vulnerability,passed,passed,passed,passed,,,, |
| fix-git,failed,passed,passed,passed,verification_failed,,, |
| fix-ocaml-gc,passed,passed,passed,passed,,,, |
| gcode-to-text,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| git-leak-recovery,passed,passed,passed,passed,,,, |
| git-multibranch,passed,passed,passed,passed,,,, |
| gpt2-codegolf,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| headless-terminal,passed,passed,passed,passed,,,, |
| hf-model-inference,passed,passed,passed,passed,,,, |
| install-windows-3.11,passed,passed,passed,failed,,,,verification_failed |
| kv-store-grpc,passed,failed,passed,failed,,verification_failed,,verification_failed |
| large-scale-text-editing,passed,passed,passed,passed,,,, |
| largest-eigenval,failed,passed,failed,passed,budget_exhausted,,budget_exhausted, |
| llm-inference-batching-scheduler,passed,passed,passed,passed,,,, |
| log-summary-date-ranges,passed,passed,passed,passed,,,, |
| mailman,passed,passed,passed,passed,,,, |
| make-doom-for-mips,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| make-mips-interpreter,failed,passed,failed,failed,verification_failed,,budget_exhausted,budget_exhausted |
| mcmc-sampling-stan,passed,passed,passed,failed,,,,verification_failed |
| merge-diff-arc-agi-task,passed,passed,passed,passed,,,, |
| model-extraction-relu-logits,passed,passed,failed,passed,,,budget_exhausted, |
| modernize-scientific-stack,passed,passed,passed,passed,,,, |
| mteb-leaderboard,failed,passed,passed,passed,verification_failed,,, |
| mteb-retrieve,passed,passed,passed,failed,,,,verification_failed |
| multi-source-data-merger,passed,passed,passed,passed,,,, |
| nginx-request-logging,passed,passed,passed,passed,,,, |
| openssl-selfsigned-cert,passed,passed,passed,passed,,,, |
| overfull-hbox,passed,passed,passed,failed,,,,budget_exhausted |
| password-recovery,passed,passed,failed,passed,,,budget_exhausted, |
| path-tracing,passed,passed,failed,passed,,,budget_exhausted, |
| path-tracing-reverse,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| polyglot-c-py,passed,passed,passed,failed,,,,verification_failed |
| polyglot-rust-c,passed,passed,passed,passed,,,, |
| portfolio-optimization,passed,passed,passed,passed,,,, |
| protein-assembly,passed,failed,failed,passed,,verification_failed,budget_exhausted, |
| prove-plus-comm,passed,passed,passed,passed,,,, |
| pypi-server,passed,passed,passed,failed,,,,verification_failed |
| pytorch-model-cli,failed,failed,failed,passed,verification_failed,verification_failed,verification_failed, |
| pytorch-model-recovery,failed,passed,passed,unscored,verification_failed,,,plumbing_failed |
| qemu-alpine-ssh,passed,failed,passed,failed,,verification_failed,,budget_exhausted |
| qemu-startup,passed,passed,passed,failed,,,,verification_failed |
| query-optimize,passed,passed,passed,failed,,,,verification_failed |
| raman-fitting,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| regex-chess,passed,failed,failed,unscored,,verification_failed,budget_exhausted,infra_failed |
| regex-log,passed,passed,passed,passed,,,, |
| reshard-c4-data,passed,passed,passed,passed,,,, |
| rstan-to-pystan,passed,passed,passed,passed,,,, |
| sam-cell-seg,passed,passed,failed,passed,,,verification_failed, |
| sanitize-git-repo,passed,failed,passed,passed,,verification_failed,, |
| schemelike-metacircular-eval,passed,passed,failed,failed,,,budget_exhausted,verification_failed |
| sparql-university,passed,passed,passed,passed,,,, |
| sqlite-db-truncate,passed,passed,passed,passed,,,, |
| sqlite-with-gcov,passed,passed,passed,passed,,,, |
| torch-pipeline-parallelism,failed,passed,failed,failed,budget_exhausted,,verification_failed,budget_exhausted |
| torch-tensor-parallelism,failed,failed,failed,failed,verification_failed,budget_exhausted,verification_failed,verification_failed |
| train-fasttext,failed,failed,failed,failed,budget_exhausted,budget_exhausted,budget_exhausted,budget_exhausted |
| tune-mjcf,passed,passed,failed,passed,,,budget_exhausted, |
| video-processing,failed,failed,failed,failed,verification_failed,verification_failed,verification_failed,verification_failed |
| vulnerable-secret,passed,passed,passed,passed,,,, |
| winning-avg-corewars,passed,passed,failed,passed,,,budget_exhausted, |
| write-compressor,failed,passed,failed,failed,budget_exhausted,,budget_exhausted,verification_failed |
