{ "benchmark": "Terminal-Bench 2 difficult subset operational probe", "run_at": "2026-07-08", "dataset": "terminal-bench/terminal-bench-2", "agent": "terminus-2", "host": "Forge/MS-01 via Harbor", "tasks": [ "cancel-async-tasks", "write-compressor", "feal-differential-cryptanalysis" ], "oracle_smoke": "passed for all three selected tasks", "important_caveat": "Small difficult subset; not leaderboard-equivalent Terminal-Bench 2.1 score. Grok 4.5 FEAL should be retried once before using as final capability claim because it hit APIError. The Grok 4.3 FEAL wrapper was interrupted with exit code 130 after Harbor had been running that task for 30m53s; publish it as interrupted/incomplete, not a scored capability failure.", "summary": { "GPT-5.5": { "passes": 3, "fails": 0, "exceptions": 0, "interrupted_incomplete": 0, "attempted_tasks": 3, "model_slug": "openai/gpt-5.5", "known_cost_usd": 0.601997 }, "Grok 4.5": { "passes": 1, "fails": 1, "exceptions": 1, "interrupted_incomplete": 0, "attempted_tasks": 3, "model_slug": "xai/grok-4.5", "known_cost_usd": 0 }, "Grok 4.3": { "passes": 1, "fails": 0, "exceptions": 1, "interrupted_incomplete": 1, "attempted_tasks": 3, "model_slug": "xai/grok-4.3", "known_cost_usd": 0.67763 } }, "rows": [ { "model": "GPT-5.5", "model_slug": "openai/gpt-5.5", "task": "cancel-async-tasks", "job": "gpt55-t2-cancel-async-tasks", "status": "pass", "reward": 1.0, "exceptions": {}, "started_at": "2026-07-08T14:40:22.250553", "finished_at": "2026-07-08T14:41:37.448516", "updated_at": "2026-07-08T14:41:37.448516", "duration_s": 75.2, "cost_usd": 0.06160000000000001, "input_tokens": 2696, "output_tokens": 1604, "cache_tokens": 0, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "GPT-5.5", "model_slug": "openai/gpt-5.5", "task": "write-compressor", "job": "gpt55-t2-write-compressor", "status": "pass", "reward": 1.0, "exceptions": {}, "started_at": "2026-07-08T14:47:06.750559", "finished_at": "2026-07-08T14:49:14.117677", "updated_at": "2026-07-08T14:49:14.117677", "duration_s": 127.4, "cost_usd": 0.21222000000000002, "input_tokens": 14910, "output_tokens": 4589, "cache_tokens": 0, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "GPT-5.5", "model_slug": "openai/gpt-5.5", "task": "feal-differential-cryptanalysis", "job": "gpt55-t2-feal-differential-cryptanalysis", "status": "pass", "reward": 1.0, "exceptions": {}, "started_at": "2026-07-08T14:49:16.375618", "finished_at": "2026-07-08T14:52:42.747981", "updated_at": "2026-07-08T14:52:42.747981", "duration_s": 206.4, "cost_usd": 0.328177, "input_tokens": 40931, "output_tokens": 8303, "cache_tokens": 27904, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "Grok 4.5", "model_slug": "xai/grok-4.5", "task": "cancel-async-tasks", "job": "grok45-t2-cancel-async-tasks", "status": "fail", "reward": 0.0, "exceptions": {}, "started_at": "2026-07-08T14:42:58.196333", "finished_at": "2026-07-08T14:45:00.181009", "updated_at": "2026-07-08T14:45:00.181009", "duration_s": 122.0, "cost_usd": null, "input_tokens": 12406, "output_tokens": 6464, "cache_tokens": 7936, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "Grok 4.5", "model_slug": "xai/grok-4.5", "task": "write-compressor", "job": "grok45-t2-write-compressor", "status": "pass", "reward": 1.0, "exceptions": {}, "started_at": "2026-07-08T14:52:44.655937", "finished_at": "2026-07-08T15:00:01.427624", "updated_at": "2026-07-08T15:00:01.427624", "duration_s": 436.8, "cost_usd": null, "input_tokens": 69737, "output_tokens": 28080, "cache_tokens": 50816, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "Grok 4.5", "model_slug": "xai/grok-4.5", "task": "feal-differential-cryptanalysis", "job": "grok45-t2-feal-differential-cryptanalysis", "status": "exception", "reward": 0.0, "exceptions": { "APIError": [ "feal-differential-cryptanalysis__T3RGNDj" ] }, "started_at": "2026-07-08T15:00:03.344035", "finished_at": "2026-07-08T15:02:01.151852", "updated_at": "2026-07-08T15:02:01.151852", "duration_s": 117.8, "cost_usd": null, "input_tokens": 3044, "output_tokens": 4808, "cache_tokens": 896, "completed_trials": 1, "errored_trials": 1, "running_trials": 0 }, { "model": "Grok 4.3", "model_slug": "xai/grok-4.3", "task": "cancel-async-tasks", "job": "grok43-t2-cancel-async-tasks", "status": "pass", "reward": 1.0, "exceptions": {}, "started_at": "2026-07-08T14:41:46.288709", "finished_at": "2026-07-08T14:42:50.926405", "updated_at": "2026-07-08T14:42:50.926405", "duration_s": 64.6, "cost_usd": 0.007267900000000001, "input_tokens": 6164, "output_tokens": 981, "cache_tokens": 2752, "completed_trials": 1, "errored_trials": 0, "running_trials": 0 }, { "model": "Grok 4.3", "model_slug": "xai/grok-4.3", "task": "write-compressor", "job": "grok43-t2-write-compressor", "status": "exception", "reward": 0.0, "exceptions": { "AgentTimeoutError": [ "write-compressor__QGaWZJL" ] }, "started_at": "2026-07-08T15:02:03.251531", "finished_at": "2026-07-08T15:17:42.645966", "updated_at": "2026-07-08T15:17:42.645966", "duration_s": 939.4, "cost_usd": 0.47044080000000005, "input_tokens": 664534, "output_tokens": 44365, "cache_tokens": 448704, "completed_trials": 1, "errored_trials": 1, "running_trials": 0 }, { "model": "Grok 4.3", "model_slug": "xai/grok-4.3", "task": "feal-differential-cryptanalysis", "job": "grok43-t2-feal-differential-cryptanalysis", "status": "interrupted_incomplete", "reward": null, "exceptions": { "Interrupted": [ "wrapper_exit_130_after_30m53s" ] }, "started_at": "2026-07-08T15:17:44.608987", "finished_at": null, "updated_at": "2026-07-08T20:48:37.799353Z", "duration_s": 1853.0, "cost_usd": 0.19992105000000004, "input_tokens": 395329, "output_tokens": 17134, "cache_tokens": 321024, "completed_trials": 1, "errored_trials": 1, "running_trials": 0, "wrapper_exit_code": 130, "note": "Wrapper interrupted after Harbor progress showed 30m53s elapsed; no completed scorer result should be treated as final." } ] }