{
 "generated": "2026-10-20 07:51 UTC",
 "rows": [
  {
   "config": "claude-sonnet-5",
   "name": "Claude Sonnet 5 \u00b7 Bedrock",
   "quant": "-",
   "baseline": true,
   "graded": 51,
   "diag": 58.8235294117647,
   "mit": 64.70588235294117,
   "e2e": 45.09803921568628,
   "e2e_k": 23,
   "e2e_ci95": [
    32.267305011132244,
    58.61552845835077
   ],
   "ttd": 315.8646620362997,
   "ttm": 399.751721834143,
   "tokens": 2967499.1458333335,
   "cache_hit_pct": 98.08798590731608,
   "call_latency_p50": 4.539996862411499,
   "call_latency_p95": 24.204505920410156,
   "ttft_p50": 0.00019407272338867188,
   "output_tps_p50": 102.11847391213666,
   "cost_per_run": 1.2151796498039216,
   "cost_total": 61.974162140000004,
   "call_errors": 0,
   "reruns": 10,
   "absorbed_429_pct": 0.0,
   "agent_min_mean": 6.466430208333334,
   "agent_min_median": 4.362683333333333,
   "input_tokens_per_run": 3272692.0392156863,
   "output_tokens_per_run": 30119.392156862745,
   "rate_limited_reruns": 0,
   "incomplete_after_reruns": 3,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": false,
    "duplicate_pvc_mounts_social_network/a1": false,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": false,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": false,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": true,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": false,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": true,
    "kafka_poison_pill_hol_block/a3": true,
    "mutating_webhook_resource_limits_social_network/a1": false,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": false,
    "namespace_memory_limit/a2": false,
    "namespace_memory_limit/a3": false,
    "network_policy_block/a1": false,
    "network_policy_block/a2": false,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": true,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": false,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": true,
    "service_wrong_pod_selection_hotel_reservation/a2": true,
    "service_wrong_pod_selection_hotel_reservation/a3": false,
    "unschedulable_incorrect_port_assignment/a1": true,
    "unschedulable_incorrect_port_assignment/a2": false,
    "unschedulable_incorrect_port_assignment/a3": false,
    "valkey_auth_disruption/a1": false,
    "valkey_auth_disruption/a2": false,
    "valkey_auth_disruption/a3": false
   },
   "price": {
    "input": 2.2,
    "output": 11.0,
    "cache_read": 0.22,
    "cache_write": 2.75,
    "source": "AWS Price List API, AmazonBedrockFoundationModels us-east-1, Claude Sonnet 5 *_standard (us. inference profile); cache write = 5-minute TTL, which Claude Code used (ephemeral_5m)"
   },
   "spend_total": null,
   "notes": {
    "route": "AWS Bedrock \u00b7 us.anthropic.claude-sonnet-5",
    "adapt": [
     "Same harness and incidents as the Kimi K3 rows, with stock Claude Code settings: token counter on, all 21 tools, judged by Sonnet 5."
    ],
    "insight": "Scored 45.1% E2E success, the same as the public SREGym leaderboard result for Claude Code + Claude Sonnet 5, which is consistent with a correctly working harness."
   },
   "vendor": {
    "limit": "",
    "source": "",
    "parallel": 6,
    "serving": "Serverless, us. inference profile"
   }
  },
  {
   "config": "kimi-k3-fireworks",
   "name": "Fireworks",
   "quant": "FP8",
   "baseline": false,
   "graded": 51,
   "diag": 82.3529411764706,
   "mit": 76.47058823529412,
   "e2e": 66.66666666666667,
   "e2e_k": 34,
   "e2e_ci95": [
    52.96812196648057,
    78.03024423727717
   ],
   "ttd": 439.64602572917937,
   "ttm": 578.5747164773941,
   "tokens": 914324.5102040817,
   "cache_hit_pct": 96.09327885354828,
   "call_latency_p50": 11.223735094070435,
   "call_latency_p95": 37.76617217063904,
   "ttft_p50": 3.3572638034820557,
   "output_tps_p50": 30.38841133505312,
   "cost_per_run": 0.6595762352941177,
   "cost_total": 33.638388,
   "call_errors": 0,
   "reruns": 6,
   "absorbed_429_pct": 0.0,
   "agent_min_mean": 9.588308503401361,
   "agent_min_median": 7.467333333333333,
   "input_tokens_per_run": 966610.8431372549,
   "output_tokens_per_run": 17489.92156862745,
   "rate_limited_reruns": 0,
   "incomplete_after_reruns": 1,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": false,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": false,
    "duplicate_pvc_mounts_social_network/a1": false,
    "duplicate_pvc_mounts_social_network/a2": true,
    "duplicate_pvc_mounts_social_network/a3": false,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": true,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": true,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": true,
    "kafka_poison_pill_hol_block/a3": false,
    "mutating_webhook_resource_limits_social_network/a1": true,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": true,
    "namespace_memory_limit/a2": false,
    "namespace_memory_limit/a3": false,
    "network_policy_block/a1": true,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": true,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": false,
    "service_wrong_pod_selection_hotel_reservation/a2": true,
    "service_wrong_pod_selection_hotel_reservation/a3": true,
    "unschedulable_incorrect_port_assignment/a1": false,
    "unschedulable_incorrect_port_assignment/a2": true,
    "unschedulable_incorrect_port_assignment/a3": true,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": true
   },
   "price": {
    "input": 3.0,
    "output": 15.0,
    "cache_read": 0.3,
    "quantization": "FP8 (Fireworks model API defaultPrecision)",
    "source": "fireworks.ai/models/fireworks/kimi-k3 serverless pricing, fetched 2026-10-18"
   },
   "spend_total": 68.8171665000001,
   "notes": {
    "route": "api.fireworks.ai/inference/v1 \u00b7 accounts/fireworks/models/kimi-k3",
    "adapt": [
     "Called through a Chat Completions route that passes the model's earlier reasoning back each turn, as Kimi K3 requires.",
     "6 agents in parallel, no rate-limited calls."
    ],
    "insight": "Among the cheapest per run, because cached tokens cost $0.30 per 1M. Also the slowest output speed, so the agent takes longest to fix each incident."
   },
   "vendor": {
    "limit": "7.2M prompt (1.8M uncached) + 72K output tokens / min",
    "source": "API response headers",
    "parallel": 6,
    "serving": "Serverless",
    "quant_source": "model API (defaultPrecision)"
   }
  },
  {
   "config": "kimi-k3-nebius",
   "name": "Nebius",
   "quant": "FP4",
   "baseline": false,
   "graded": 51,
   "diag": 76.47058823529412,
   "mit": 78.43137254901961,
   "e2e": 66.66666666666667,
   "e2e_k": 34,
   "e2e_ci95": [
    52.96812196648057,
    78.03024423727717
   ],
   "ttd": 200.91037819899765,
   "ttm": 325.2678188529669,
   "tokens": 1466519.0588235294,
   "cache_hit_pct": 97.24507281328003,
   "call_latency_p50": 3.112420082092285,
   "call_latency_p95": 13.379302978515625,
   "ttft_p50": 0.8256950378417969,
   "output_tps_p50": 114.65909469437139,
   "cost_per_run": 4.7186,
   "cost_total": 240.6486,
   "call_errors": 0,
   "reruns": 1,
   "absorbed_429_pct": 0.0,
   "agent_min_mean": 5.123900980392157,
   "agent_min_median": 2.9665166666666667,
   "input_tokens_per_run": 1463796.7647058824,
   "output_tokens_per_run": 21813.980392156864,
   "rate_limited_reruns": 0,
   "incomplete_after_reruns": 0,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": true,
    "duplicate_pvc_mounts_social_network/a1": false,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": false,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": false,
    "finalizer_deadlock_controller_hotel_reservation/a1": false,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": true,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": false,
    "kafka_poison_pill_hol_block/a3": false,
    "mutating_webhook_resource_limits_social_network/a1": true,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": false,
    "namespace_memory_limit/a2": true,
    "namespace_memory_limit/a3": true,
    "network_policy_block/a1": true,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": true,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": true,
    "service_wrong_pod_selection_hotel_reservation/a2": true,
    "service_wrong_pod_selection_hotel_reservation/a3": true,
    "unschedulable_incorrect_port_assignment/a1": true,
    "unschedulable_incorrect_port_assignment/a2": false,
    "unschedulable_incorrect_port_assignment/a3": true,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": true
   },
   "price": {
    "input": 3.0,
    "output": 15.0,
    "cache_read": null,
    "quantization": "fp4",
    "region": "eu-west2 (FR)",
    "source": "GET api.studio.nebius.com/v1/models?verbose=true pricing"
   },
   "spend_total": 300.4909560000002,
   "notes": {
    "route": "api.studio.nebius.com/v1 \u00b7 moonshotai/Kimi-K3 \u00b7 FP4",
    "adapt": [
     "Native Chat Completions route; passes the model's earlier reasoning back each turn.",
     "4 agents in parallel, no rate-limited calls."
    ],
    "insight": "Fastest provider: lowest time to first token, highest output speed and shortest time to fix. 97% of prompt tokens were cache hits, but Nebius bills cached tokens at the full $3 input price, so a run costs about 7\u00d7 as much as on the cheapest provider."
   },
   "vendor": {
    "limit": "2M tokens + 1,000 requests / min",
    "source": "API response headers",
    "parallel": 4,
    "serving": "Serverless",
    "quant_source": "models API"
   }
  },
  {
   "config": "kimi-k3-baseten",
   "name": "Baseten",
   "quant": "FP8",
   "baseline": false,
   "graded": 51,
   "diag": 74.50980392156863,
   "mit": 72.54901960784314,
   "e2e": 62.745098039215684,
   "e2e_k": 32,
   "e2e_ci95": [
    49.02496270033425,
    74.6796702790099
   ],
   "ttd": 390.11051750183105,
   "ttm": 491.11193025813384,
   "tokens": 1045202.5306122449,
   "cache_hit_pct": 94.91888421468234,
   "call_latency_p50": 6.161760091781616,
   "call_latency_p95": 23.286400079727173,
   "ttft_p50": 2.6035900115966797,
   "output_tps_p50": 56.82234924307018,
   "cost_per_run": 0.7378318823529412,
   "cost_total": 37.629426,
   "call_errors": 87,
   "reruns": 3,
   "absorbed_429_pct": 4.5288912024986985,
   "agent_min_mean": 8.172923,
   "agent_min_median": 5.718391666666667,
   "input_tokens_per_run": 1053385.7647058824,
   "output_tokens_per_run": 18126.50980392157,
   "rate_limited_reruns": 20,
   "incomplete_after_reruns": 0,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": false,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": false,
    "duplicate_pvc_mounts_social_network/a1": false,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": false,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": false,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": true,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": true,
    "kafka_poison_pill_hol_block/a3": true,
    "mutating_webhook_resource_limits_social_network/a1": false,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": true,
    "namespace_memory_limit/a2": true,
    "namespace_memory_limit/a3": false,
    "network_policy_block/a1": true,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": true,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": false,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": true,
    "service_wrong_pod_selection_hotel_reservation/a2": false,
    "service_wrong_pod_selection_hotel_reservation/a3": false,
    "unschedulable_incorrect_port_assignment/a1": false,
    "unschedulable_incorrect_port_assignment/a2": true,
    "unschedulable_incorrect_port_assignment/a3": true,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": true
   },
   "price": {
    "input": 3.0,
    "output": 15.0,
    "cache_read": 0.3,
    "quantization": "fp8",
    "source": "GET inference.baseten.co/v1/models pricing"
   },
   "spend_total": 60.25782179999994,
   "notes": {
    "route": "inference.baseten.co/v1 \u00b7 moonshotai/Kimi-K3 \u00b7 FP8",
    "adapt": [
     "Default account tier is capped at 100,000 tokens and 15 requests per rate-limit window. One agent re-sending a 20\u201340K-token transcript every turn nearly fills it, so it runs one agent at a time. Baseten raises the limit on request."
    ],
    "insight": "Mid-range on speed and cost. The main operational limit is the account rate limit, which allows one agent at a time."
   },
   "vendor": {
    "limit": "100K tokens + 15 requests / window",
    "source": "API response headers",
    "parallel": 1,
    "serving": "Serverless",
    "quant_source": "models API"
   }
  },
  {
   "config": "kimi-k3-aws",
   "name": "AWS Bedrock",
   "quant": "not disclosed",
   "baseline": false,
   "graded": 51,
   "diag": 82.3529411764706,
   "mit": 76.47058823529412,
   "e2e": 68.62745098039215,
   "e2e_k": 35,
   "e2e_ci95": [
    54.97274248374967,
    79.67249033221485
   ],
   "ttd": 225.12138114723504,
   "ttm": 330.617929725086,
   "tokens": 1218682.9803921569,
   "cache_hit_pct": 95.73861104202497,
   "call_latency_p50": 5.126765966415405,
   "call_latency_p95": 14.02924919128418,
   "ttft_p50": 2.7357020378112793,
   "output_tps_p50": 72.80409561211819,
   "cost_per_run": 0.8723564688235295,
   "cost_total": 44.49017991,
   "call_errors": 0,
   "reruns": 1,
   "absorbed_429_pct": 0.0,
   "agent_min_mean": 5.437029738562091,
   "agent_min_median": 3.9852333333333334,
   "input_tokens_per_run": 1211015.705882353,
   "output_tokens_per_run": 19131.78431372549,
   "rate_limited_reruns": 0,
   "incomplete_after_reruns": 0,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": false,
    "duplicate_pvc_mounts_social_network/a1": true,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": true,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": false,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": false,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": true,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": false,
    "kafka_poison_pill_hol_block/a3": true,
    "mutating_webhook_resource_limits_social_network/a1": false,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": true,
    "namespace_memory_limit/a2": true,
    "namespace_memory_limit/a3": false,
    "network_policy_block/a1": false,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": true,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": true,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": true,
    "service_wrong_pod_selection_hotel_reservation/a2": false,
    "service_wrong_pod_selection_hotel_reservation/a3": true,
    "unschedulable_incorrect_port_assignment/a1": true,
    "unschedulable_incorrect_port_assignment/a2": true,
    "unschedulable_incorrect_port_assignment/a3": true,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": true
   },
   "price": {
    "input": 3.3,
    "output": 16.5,
    "cache_read": 0.33,
    "cache_write": 4.125,
    "source": "AWS Price List API, AmazonBedrock us-east-1, moonshotai.kimi-k3 *-standard (us. inference profile)"
   },
   "spend_total": 127.90252447500022,
   "notes": {
    "route": "Bedrock OpenAI-compatible endpoint (/openai/v1) \u00b7 us.moonshotai.kimi-k3",
    "adapt": [
     "Called through Bedrock's OpenAI-compatible endpoint, which passes the model's earlier reasoning back each turn; LiteLLM's Converse route did not.",
     "Bedrock rejects the temperature parameter for Kimi K3. Sampling parameters are left at model defaults on every provider."
    ],
    "insight": "Best balance of speed and cost. On the us. inference profile it is priced 10% above the others ($3.30 / $16.50; the global. profile matches $3 / $15). It is the only provider that also charges for cache writes."
   },
   "vendor": {
    "limit": "20M tokens / min (us. cross-region)",
    "source": "AWS service quota",
    "parallel": 6,
    "serving": "Serverless, us. inference profile",
    "quant_source": "not in AWS model API"
   }
  },
  {
   "config": "kimi-k3-together",
   "name": "Together AI",
   "quant": "FP4",
   "baseline": false,
   "graded": 51,
   "diag": 74.50980392156863,
   "mit": 74.50980392156863,
   "e2e": 58.8235294117647,
   "e2e_k": 30,
   "e2e_ci95": [
    45.16509274741746,
    71.24580700751311
   ],
   "ttd": 374.6069306392296,
   "ttm": 542.2077601890938,
   "tokens": 943887.0392156863,
   "cache_hit_pct": 95.52929647394745,
   "call_latency_p50": 9.108674049377441,
   "call_latency_p95": 34.63948607444763,
   "ttft_p50": 2.3988919258117676,
   "output_tps_p50": 36.02073495426545,
   "cost_per_run": 0.6469080705882353,
   "cost_total": 32.9923116,
   "call_errors": 538,
   "reruns": 2,
   "absorbed_429_pct": 23.81584772023019,
   "agent_min_mean": 9.07452843137255,
   "agent_min_median": 7.314466666666666,
   "input_tokens_per_run": 939984.5882352941,
   "output_tokens_per_run": 16391.764705882353,
   "rate_limited_reruns": 12,
   "incomplete_after_reruns": 0,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": false,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": true,
    "duplicate_pvc_mounts_social_network/a1": false,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": false,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": true,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": false,
    "internal_traffic_policy_local_astronomy_shop/a3": true,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": true,
    "kafka_poison_pill_hol_block/a3": true,
    "mutating_webhook_resource_limits_social_network/a1": false,
    "mutating_webhook_resource_limits_social_network/a2": false,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": true,
    "namespace_memory_limit/a2": false,
    "namespace_memory_limit/a3": false,
    "network_policy_block/a1": true,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": true,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": false,
    "service_wrong_pod_selection_hotel_reservation/a2": true,
    "service_wrong_pod_selection_hotel_reservation/a3": true,
    "unschedulable_incorrect_port_assignment/a1": false,
    "unschedulable_incorrect_port_assignment/a2": true,
    "unschedulable_incorrect_port_assignment/a3": false,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": false
   },
   "price": {
    "input": 3.0,
    "output": 15.0,
    "cache_read": 0.3,
    "source": "GET api.together.xyz/v1/models pricing",
    "quantization": "FP4 (Together model page)"
   },
   "spend_total": 38.26798559999996,
   "notes": {
    "route": "api.together.xyz/v1 \u00b7 moonshotai/Kimi-K3",
    "adapt": [
     "At 2 agents in parallel on our account, about a quarter of calls were rate-limited (HTTP 429) and retried, which slowed the agent.",
     "Account credit is checked against each request's full max_tokens (32K for Claude Code), so the balance needs headroom beyond actual spend."
    ],
    "insight": "Same list price as Fireworks and about the same cost per run. Frequent rate limiting makes each run longer."
   },
   "vendor": {
    "limit": "Dynamic, follows live capacity",
    "source": "Together docs; no limit headers",
    "parallel": 2,
    "serving": "Serverless",
    "quant_source": "model page"
   }
  },
  {
   "config": "kimi-k3-modal",
   "name": "Modal",
   "quant": "not disclosed",
   "baseline": false,
   "graded": 51,
   "diag": 88.23529411764706,
   "mit": 72.54901960784314,
   "e2e": 66.66666666666667,
   "e2e_k": 34,
   "e2e_ci95": [
    52.96812196648057,
    78.03024423727717
   ],
   "ttd": 283.8722552318199,
   "ttm": 439.5372721821654,
   "tokens": 1311422.96,
   "cache_hit_pct": 96.65922886267677,
   "call_latency_p50": 5.842687129974365,
   "call_latency_p95": 21.02467107772827,
   "ttft_p50": 2.2892889976501465,
   "output_tps_p50": 61.887016941306754,
   "cost_per_run": 0.8429301294117647,
   "cost_total": 42.9894366,
   "call_errors": 0,
   "reruns": 14,
   "absorbed_429_pct": 0.0,
   "agent_min_mean": 7.058421666666667,
   "agent_min_median": 4.93545,
   "input_tokens_per_run": 1316521.0980392157,
   "output_tokens_per_run": 21605.56862745098,
   "rate_limited_reruns": 1,
   "incomplete_after_reruns": 0,
   "per_problem": {
    "cronjob_sidecar_blocks_completion_hotel_reservation/a1": false,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a2": true,
    "cronjob_sidecar_blocks_completion_hotel_reservation/a3": true,
    "duplicate_pvc_mounts_social_network/a1": true,
    "duplicate_pvc_mounts_social_network/a2": false,
    "duplicate_pvc_mounts_social_network/a3": true,
    "edge_request_filter_cpu_saturation/a1": true,
    "edge_request_filter_cpu_saturation/a2": true,
    "edge_request_filter_cpu_saturation/a3": true,
    "env_variable_shadowing_astronomy_shop/a1": true,
    "env_variable_shadowing_astronomy_shop/a2": true,
    "env_variable_shadowing_astronomy_shop/a3": true,
    "finalizer_deadlock_controller_hotel_reservation/a1": true,
    "finalizer_deadlock_controller_hotel_reservation/a2": true,
    "finalizer_deadlock_controller_hotel_reservation/a3": true,
    "internal_traffic_policy_local_astronomy_shop/a1": true,
    "internal_traffic_policy_local_astronomy_shop/a2": true,
    "internal_traffic_policy_local_astronomy_shop/a3": false,
    "kafka_poison_pill_hol_block/a1": true,
    "kafka_poison_pill_hol_block/a2": false,
    "kafka_poison_pill_hol_block/a3": false,
    "mutating_webhook_resource_limits_social_network/a1": false,
    "mutating_webhook_resource_limits_social_network/a2": true,
    "mutating_webhook_resource_limits_social_network/a3": false,
    "namespace_memory_limit/a1": false,
    "namespace_memory_limit/a2": false,
    "namespace_memory_limit/a3": true,
    "network_policy_block/a1": true,
    "network_policy_block/a2": true,
    "network_policy_block/a3": true,
    "rolling_update_misconfigured_social_network/a1": false,
    "rolling_update_misconfigured_social_network/a2": false,
    "rolling_update_misconfigured_social_network/a3": false,
    "search_rate_retry_collapse_hotel_reservation/a1": false,
    "search_rate_retry_collapse_hotel_reservation/a2": false,
    "search_rate_retry_collapse_hotel_reservation/a3": false,
    "secret_rotation_stale_env_credentials_astronomy_shop/a1": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a2": true,
    "secret_rotation_stale_env_credentials_astronomy_shop/a3": false,
    "service_dns_resolution_failure_social_network/a1": true,
    "service_dns_resolution_failure_social_network/a2": true,
    "service_dns_resolution_failure_social_network/a3": true,
    "service_wrong_pod_selection_hotel_reservation/a1": true,
    "service_wrong_pod_selection_hotel_reservation/a2": true,
    "service_wrong_pod_selection_hotel_reservation/a3": true,
    "unschedulable_incorrect_port_assignment/a1": true,
    "unschedulable_incorrect_port_assignment/a2": true,
    "unschedulable_incorrect_port_assignment/a3": false,
    "valkey_auth_disruption/a1": true,
    "valkey_auth_disruption/a2": true,
    "valkey_auth_disruption/a3": true
   },
   "price": {
    "input": 3.0,
    "output": 15.0,
    "cache_read": 0.3,
    "source": "modal.com/endpoints create page, Shared Endpoint pricing for moonshotai/Kimi-K3, fetched 2026-10-19"
   },
   "spend_total": 51.99096479999992,
   "notes": {
    "route": "Modal Shared Endpoint (us-west) \u00b7 moonshotai/Kimi-K3",
    "adapt": [
     "Serverless per-token billing; the endpoint is created once in the workspace and called with a workspace proxy token.",
     "Each run sends its own Modal-Routing-Affinity-Key header, as Modal's docs ask for agent loops, so a session stays on one warm prompt cache."
    ],
    "insight": "Under parallel load the shared endpoint ended some streamed responses early (finish_reason \"abort\"), which cut off the agent's tool call; 13 runs stopped early this way and were re-run, and all finished within 3 tries."
   },
   "vendor": {
    "limit": "16 concurrent requests / workspace",
    "source": "Modal endpoint page",
    "parallel": 4,
    "serving": "Shared endpoint, us-west",
    "quant_source": "not published"
   }
  }
 ],
 "problems": [
  {
   "id": "cronjob_sidecar_blocks_completion_hotel_reservation",
   "category": "Misconfiguration Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "duplicate_pvc_mounts_social_network",
   "category": "Cloud Management System Failure",
   "app": "social-network"
  },
  {
   "id": "edge_request_filter_cpu_saturation",
   "category": "System/Application Software Failure",
   "app": "astronomy-shop"
  },
  {
   "id": "env_variable_shadowing_astronomy_shop",
   "category": "Misconfiguration Failure",
   "app": "astronomy-shop"
  },
  {
   "id": "finalizer_deadlock_controller_hotel_reservation",
   "category": "Security Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "internal_traffic_policy_local_astronomy_shop",
   "category": "Network Failure",
   "app": "astronomy-shop"
  },
  {
   "id": "kafka_poison_pill_hol_block",
   "category": "System/Application Software Failure",
   "app": "astronomy-shop"
  },
  {
   "id": "mutating_webhook_resource_limits_social_network",
   "category": "Misconfiguration Failure",
   "app": "social-network"
  },
  {
   "id": "namespace_memory_limit",
   "category": "Cloud Management System Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "network_policy_block",
   "category": "Network Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "rolling_update_misconfigured_social_network",
   "category": "Misconfiguration Failure",
   "app": "social-network"
  },
  {
   "id": "search_rate_retry_collapse_hotel_reservation",
   "category": "Metastable Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "secret_rotation_stale_env_credentials_astronomy_shop",
   "category": "Security Failure",
   "app": "astronomy-shop"
  },
  {
   "id": "service_dns_resolution_failure_social_network",
   "category": "Network Failure",
   "app": "social-network"
  },
  {
   "id": "service_wrong_pod_selection_hotel_reservation",
   "category": "Misconfiguration Failure",
   "app": "hotel-reservation"
  },
  {
   "id": "unschedulable_incorrect_port_assignment",
   "category": "Multiple Independent Failures",
   "app": "astronomy-shop"
  },
  {
   "id": "valkey_auth_disruption",
   "category": "Security Failure",
   "app": "astronomy-shop"
  }
 ],
 "leaderboard": {
  "agent": "Claude Code",
  "model": "Claude Sonnet 5",
  "diag": 54.9,
  "mit": 62.7,
  "e2e": 45.1,
  "ttd": 308.6,
  "ttm": 505.2
 }
}