{
  "schema_version": 1,
  "status": "completed_with_model_identity_deviation",
  "experiment_id": "20260827T231837Z",
  "experiment_question": "We gave Claude and Codex the exact same raw experiment evidence. Which one made the result easier to understand?",
  "source_experiment": {
    "run": "runs/20260826T232559Z/",
    "results_sha256": "062483c120974664a990429456276cb67ff03eabb61feb9f71abfae135b45a33",
    "question": "Can Claude Code and Codex build a polished SaaS landing page from the same one shot brief?"
  },
  "exact_contestant_prompt": "Turn the supplied raw experiment evidence into a genuinely excellent WhichBuildWon case-study page.\n\nThe person opening the page should be able to understand within roughly 10 seconds:\n\n1. what question was tested\n2. what Claude actually built\n3. what Codex actually built\n4. the most meaningful differences between the results\n5. what the experiment suggests, with appropriate scope\n\nUse only evidence recorded in the supplied experiment. Never invent facts, explanations, motivations, test results, or missing data.\n\nThe actual builds and screenshots are primary evidence and should be treated as such.\n\nMake the page useful and interesting to a human deciding how to use AI coding agents.\n\nYou have discretion over hierarchy, composition, presentation, copy, evidence selection, comparison treatment, and page-specific visual design.\n\nPreserve the site's global identity and navigation, but you may substantially redesign this case-study page.\n\nDeterministic checks and scores are supporting evidence. Decide how prominently they deserve to appear based on what is actually useful to the reader.\n\nDo not modify unrelated routes or experiment evidence.\n\nDo not add external facts or perform web research.\n\nBuild the page, ensure it works at desktop and mobile widths, run the appropriate existing checks, and stop after this implementation pass.\n",
  "prompt_sha256": "0379852d2fd33a0d385a5276d344d6c1acdd5ce7dbc9a9bf52dbfc64a602178b",
  "prompt_bytes": 1348,
  "fixture": {
    "commit": "3fe30b6e152649b812f4b6ead9aac6786d9a6b83",
    "clean": true,
    "same_start_for_both": true,
    "evidence_byte_identical": true,
    "source_results_sha256": "062483c120974664a990429456276cb67ff03eabb61feb9f71abfae135b45a33"
  },
  "contestants": {
    "claude": {
      "agent": "Claude Code",
      "agent_version": "2.1.141 (Claude Code)",
      "requested_product_model": "Opus 5",
      "configured_model": "opus",
      "configured_effort": "high",
      "confirmed_runtime_models": [
        "claude-haiku-4-5-20251001",
        "claude-opus-4-7"
      ],
      "primary_runtime_model": "claude-opus-4-7",
      "auxiliary_runtime_model": "claude-haiku-4-5-20251001",
      "model_identity_status": "CLI output confirms claude-opus-4-7, not the requested Opus 5 label",
      "primary_implementation_invocations": 1,
      "pre_model_failures": 2,
      "started_at": "2026-08-27T23:43:12.372Z",
      "finished_at": "2026-08-27T23:50:18.004Z",
      "duration_ms": 425632,
      "exit_status": {
        "exit_code": 0,
        "signal": null,
        "spawn_error": null
      },
      "usage": {
        "input_tokens": 76,
        "cache_creation_input_tokens": 183740,
        "cache_read_input_tokens": 3261355,
        "output_tokens": 24077,
        "server_tool_use": {
          "web_search_requests": 0,
          "web_fetch_requests": 0
        },
        "service_tier": "standard",
        "cache_creation": {
          "ephemeral_1h_input_tokens": 183740,
          "ephemeral_5m_input_tokens": 0
        },
        "inference_geo": "",
        "iterations": [
          {
            "input_tokens": 1,
            "output_tokens": 526,
            "cache_read_input_tokens": 92076,
            "cache_creation_input_tokens": 164,
            "cache_creation": {
              "ephemeral_5m_input_tokens": 0,
              "ephemeral_1h_input_tokens": 164
            },
            "type": "message"
          }
        ],
        "speed": "standard"
      },
      "cost_usd": 3.3820515,
      "turns": 55,
      "grader": {
        "schema_version": 1,
        "workspace": "/tmp/wbw-case-study.mXxNO1",
        "started_at": "2026-08-27T23:50:18.509Z",
        "commands": {
          "install": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 413,
            "stdout": "install.stdout.log",
            "stderr": "install.stderr.log"
          },
          "build": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 3236,
            "stdout": "build.stdout.log",
            "stderr": "build.stderr.log"
          },
          "typecheck": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 1012,
            "stdout": "typecheck.stdout.log",
            "stderr": "typecheck.stderr.log"
          },
          "server": {
            "http_status_to_ready": 200,
            "stdout": "server.stdout.log",
            "stderr": "server.stderr.log"
          }
        },
        "checks": {
          "production_build_success": {
            "passed": true,
            "observed": 0,
            "detail": "npm run build"
          },
          "typescript_success": {
            "passed": true,
            "observed": 0,
            "detail": "npm run typecheck"
          },
          "desktop_page_loads": {
            "passed": true,
            "observed": 200,
            "detail": null
          },
          "desktop_one_h1": {
            "passed": true,
            "observed": 1,
            "detail": null
          },
          "desktop_global_identity": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "desktop_experiment_question_visible": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "desktop_recorded_screenshots_load": {
            "passed": true,
            "observed": [
              {
                "src": "/evidence/claude-desktop.png",
                "ok": true
              },
              {
                "src": "/evidence/codex-desktop.png",
                "ok": true
              },
              {
                "src": "/evidence/claude-mobile.png",
                "ok": true
              },
              {
                "src": "/evidence/codex-mobile.png",
                "ok": true
              }
            ],
            "detail": null
          },
          "desktop_no_horizontal_overflow": {
            "passed": true,
            "observed": 0,
            "detail": null
          },
          "mobile_page_loads": {
            "passed": true,
            "observed": 200,
            "detail": null
          },
          "mobile_one_h1": {
            "passed": true,
            "observed": 1,
            "detail": null
          },
          "mobile_global_identity": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "mobile_experiment_question_visible": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "mobile_recorded_screenshots_load": {
            "passed": true,
            "observed": [
              {
                "src": "/evidence/claude-desktop.png",
                "ok": true
              },
              {
                "src": "/evidence/codex-desktop.png",
                "ok": true
              },
              {
                "src": "/evidence/claude-mobile.png",
                "ok": true
              },
              {
                "src": "/evidence/codex-mobile.png",
                "ok": true
              }
            ],
            "detail": null
          },
          "mobile_no_horizontal_overflow": {
            "passed": true,
            "observed": 0,
            "detail": null
          },
          "no_browser_console_errors": {
            "passed": true,
            "observed": [],
            "detail": null
          },
          "desktop_screenshot_captured": {
            "passed": true,
            "observed": 1131932,
            "detail": null
          },
          "mobile_screenshot_captured": {
            "passed": true,
            "observed": 978019,
            "detail": null
          }
        },
        "console_errors": [],
        "screenshots": {
          "desktop": "screenshots/desktop.png",
          "mobile": "screenshots/mobile.png"
        },
        "finished_at": "2026-08-27T23:50:26.030Z",
        "duration_ms": 7521,
        "summary": {
          "passed": 17,
          "failed": [],
          "total": 17
        }
      },
      "source_diff": {
        "stat": "app/globals.css | 519 ++++++++++++++++++++++++++++++++++++++++++++++++-----\n app/layout.tsx  |   4 +-\n app/page.tsx    | 539 +++++++++++++++++++++++++++++++++++++++++++++++---------\n 3 files changed, 933 insertions(+), 129 deletions(-)",
        "sha256": "2cebffe2cc7c3e7fffa3d1aedeb29cd71185754a81484e49029c449b0d05762a"
      },
      "evidence": {
        "source": "claude/source/",
        "diff": "claude/final.diff",
        "git_status": "claude/final-git-status.txt",
        "desktop_screenshot": "claude/screenshots/desktop.png",
        "mobile_screenshot": "claude/screenshots/mobile.png",
        "build_log": "claude/build.stdout.log",
        "typecheck_log": "claude/typecheck.stdout.log",
        "grader": "claude/grader.json",
        "run_record": "claude/contestant-run.json",
        "stdout": "claude/contestant.stdout.log",
        "stderr": "claude/contestant.stderr.log"
      },
      "asset_sha256": {
        "desktop_screenshot": "62e4bc23cca2e292a978b612ed0575ce8329d3b34d4d92c7788eaedbe059b9f0",
        "mobile_screenshot": "829a313fa2c516e78d34e3a3530f60d0f27475f76d88e41271784bbb2bc48573"
      }
    },
    "codex": {
      "agent": "Codex",
      "agent_version": "codex-cli 0.150.0",
      "configured_model": "gpt-5.6-sol",
      "configured_effort": "high",
      "confirmed_runtime_models": [],
      "primary_runtime_model": null,
      "model_identity_status": "configured by explicit CLI argument; runtime model not exposed",
      "primary_implementation_invocations": 1,
      "pre_model_failures": 0,
      "started_at": "2026-08-27T23:25:12.808Z",
      "finished_at": "2026-08-27T23:37:48.474Z",
      "duration_ms": 755666,
      "exit_status": {
        "exit_code": 0,
        "signal": null,
        "spawn_error": null
      },
      "usage": {
        "input_tokens": 2003978,
        "cached_input_tokens": 1893632,
        "cache_write_input_tokens": 0,
        "output_tokens": 37383,
        "reasoning_output_tokens": 6981
      },
      "cost_usd": null,
      "turns": null,
      "grader": {
        "schema_version": 1,
        "workspace": "/tmp/wbw-case-study.h7M7pM",
        "started_at": "2026-08-27T23:42:41.550Z",
        "commands": {
          "install": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 428,
            "stdout": "install.stdout.log",
            "stderr": "install.stderr.log"
          },
          "build": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 3127,
            "stdout": "build.stdout.log",
            "stderr": "build.stderr.log"
          },
          "typecheck": {
            "exit_code": 0,
            "signal": null,
            "error": null,
            "duration_ms": 648,
            "stdout": "typecheck.stdout.log",
            "stderr": "typecheck.stderr.log"
          },
          "server": {
            "http_status_to_ready": 200,
            "stdout": "server.stdout.log",
            "stderr": "server.stderr.log"
          }
        },
        "checks": {
          "production_build_success": {
            "passed": true,
            "observed": 0,
            "detail": "npm run build"
          },
          "typescript_success": {
            "passed": true,
            "observed": 0,
            "detail": "npm run typecheck"
          },
          "desktop_page_loads": {
            "passed": true,
            "observed": 200,
            "detail": null
          },
          "desktop_one_h1": {
            "passed": true,
            "observed": 1,
            "detail": null
          },
          "desktop_global_identity": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "desktop_experiment_question_visible": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "desktop_recorded_screenshots_load": {
            "passed": true,
            "observed": [
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fclaude-desktop.0ypw_l4jr4fzt.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fclaude-mobile.3554f3rgxnuub.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fcodex-desktop.3bqhzfsar_u7i.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fcodex-mobile.355m3_f03vl-q.png&w=3840&q=75",
                "ok": true
              }
            ],
            "detail": null
          },
          "desktop_no_horizontal_overflow": {
            "passed": true,
            "observed": 0,
            "detail": null
          },
          "mobile_page_loads": {
            "passed": true,
            "observed": 200,
            "detail": null
          },
          "mobile_one_h1": {
            "passed": true,
            "observed": 1,
            "detail": null
          },
          "mobile_global_identity": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "mobile_experiment_question_visible": {
            "passed": true,
            "observed": true,
            "detail": null
          },
          "mobile_recorded_screenshots_load": {
            "passed": true,
            "observed": [
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fclaude-desktop.0ypw_l4jr4fzt.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fclaude-mobile.3554f3rgxnuub.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fcodex-desktop.3bqhzfsar_u7i.png&w=3840&q=75",
                "ok": true
              },
              {
                "src": "/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Fcodex-mobile.355m3_f03vl-q.png&w=3840&q=75",
                "ok": true
              }
            ],
            "detail": null
          },
          "mobile_no_horizontal_overflow": {
            "passed": true,
            "observed": 0,
            "detail": null
          },
          "no_browser_console_errors": {
            "passed": true,
            "observed": [],
            "detail": null
          },
          "desktop_screenshot_captured": {
            "passed": true,
            "observed": 988408,
            "detail": null
          },
          "mobile_screenshot_captured": {
            "passed": true,
            "observed": 698492,
            "detail": null
          }
        },
        "console_errors": [],
        "screenshots": {
          "desktop": "screenshots/desktop.png",
          "mobile": "screenshots/mobile.png"
        },
        "finished_at": "2026-08-27T23:42:48.213Z",
        "duration_ms": 6663,
        "summary": {
          "passed": 17,
          "failed": [],
          "total": 17
        }
      },
      "source_diff": {
        "stat": "app/globals.css | 578 +++++++++++++++++++++++++++++++++++++++++++++++++++-----\n app/layout.tsx  |   4 +-\n app/page.tsx    | 398 +++++++++++++++++++++++++++++---------\n 3 files changed, 844 insertions(+), 136 deletions(-)",
        "sha256": "d3501c7a9dfdd2921049bc69a25905f9a13d73d61f4dd947daf1d6d763cf502f"
      },
      "evidence": {
        "source": "codex/source/",
        "diff": "codex/final.diff",
        "git_status": "codex/final-git-status.txt",
        "desktop_screenshot": "codex/screenshots/desktop.png",
        "mobile_screenshot": "codex/screenshots/mobile.png",
        "build_log": "codex/build.stdout.log",
        "typecheck_log": "codex/typecheck.stdout.log",
        "grader": "codex/grader.json",
        "run_record": "codex/contestant-run.json",
        "stdout": "codex/contestant.stdout.log",
        "stderr": "codex/contestant.stderr.log"
      },
      "asset_sha256": {
        "desktop_screenshot": "3f3e86ba72fb7a074b0e265620195f5cc7adf4e8a672ef1f5daf422bf391ee0b",
        "mobile_screenshot": "65fd3bcb66aa9f29a6d401dc705e75b34b6394e6c3e03f238299c2282eb6e37a"
      }
    }
  },
  "mechanical_result": {
    "same_grader_check_names": true,
    "claude": {
      "passed": 17,
      "failed": [],
      "total": 17
    },
    "codex": {
      "passed": 17,
      "failed": [],
      "total": 17
    },
    "desktop_and_mobile_http_200": true,
    "all_recorded_source_screenshots_loaded": true,
    "horizontal_overflow_px": {
      "claude": {
        "desktop": 0,
        "mobile": 0
      },
      "codex": {
        "desktop": 0,
        "mobile": 0
      }
    },
    "browser_console_errors": {
      "claude": [],
      "codex": []
    }
  },
  "evaluation": {
    "result": "Codex made the source experiment easier to understand in this run: its shorter page used a stronger headline-to-question-to-large-screenshot sequence and a more concise scoped takeaway, while Claude exposed more raw traceability but made the reader work through a longer, denser comparison.",
    "clearly_stronger_for_user_job": "codex",
    "criteria": {
      "ten_second_comprehension": {
        "stronger": "codex",
        "observation": "Codex places a large summary headline, a one-paragraph contrast, a scoped result card, and the exact source-experiment question before the build section. Claude also exposes the exact question near the top, but at smaller scale inside a two-cell information panel."
      },
      "actual_build_screenshot_use": {
        "stronger": "codex",
        "observation": "Both pages show the two recorded desktop captures side by side near the top. Codex gives the captures taller browser frames and labels them as primary evidence; Claude uses smaller cards and surrounds them with more metadata."
      },
      "what_happened_and_meaningful_differences": {
        "stronger": "codex",
        "observation": "Codex curates four reader-facing differences—visual direction, feature story, trust pattern, and mobile treatment—then separates mechanical checks. Claude supplies six detailed rows, including CSS and files touched, which improves implementation traceability but dilutes the product/editorial comparison."
      },
      "evidence_vs_interpretation": {
        "stronger": "codex",
        "observation": "Codex separates the screenshot comparison, recorded checks, takeaway, scope note, and evidence disclosure. Claude labels its evidence sections clearly, but later calls placeholder links idiomatic in agency work and states that every hard requirement was met even though no-broken-links was a hard requirement and failed."
      },
      "unsupported_claims": {
        "stronger": "codex",
        "observation": "No material unsupported factual claim was identified in Codex’s page. Claude introduced the unrecorded generalization that placeholder links are idiomatic in agency work and an overbroad all-hard-requirements claim."
      },
      "scoped_takeaway": {
        "stronger": "codex",
        "observation": "Codex ends the main narrative with a concise statement that Codex followed testable requirements more completely while screenshots leave taste to the reader, followed immediately by a one-task/one-run limitation. Claude’s four-card interpretation is more extensive but less concise."
      },
      "visual_hierarchy_and_information_design": {
        "stronger": "codex",
        "observation": "Codex uses fewer, larger typographic decisions and a shorter page; its primary output comparison reads as one strong editorial unit. Its main weakness is giving a 20/20 score card prominent above-the-fold space, which overemphasizes supporting checks. Claude is restrained and systematic but smaller, denser, and more report-like."
      },
      "desktop_and_mobile_usability": {
        "stronger": "codex",
        "observation": "Both passed the same desktop/mobile loading, screenshot, console, and zero-overflow checks. At mobile width, Codex preserves the same hierarchy with large stacked screenshots; Claude remains usable but is materially longer and denser."
      },
      "evidence_traceability": {
        "stronger": "claude",
        "observation": "Claude links 17 raw artifacts and exposes more implementation/runtime metrics. Codex links 9 core artifacts and provides the necessary prompt, checks, diffs, validation, manifest, method, and limitations, but not the same breadth."
      }
    },
    "tradeoff": "Codex is easier to scan and understand; Claude is the more exhaustive evidence index. The Codex hero is also too score-forward for WhichBuildWon’s ideal editorial order."
  },
  "infrastructure_failures": [
    {
      "contestant": "claude",
      "stage": "orchestration before contestant invocation",
      "observed_exit_code": 1,
      "cause": "The wrapper attempted to write workspace-path.txt before creating the contestant evidence directory.",
      "model_invocation_occurred": false,
      "rerun_valid": true,
      "preserved_output": "zsh:3: no such file or directory: runs/20260827T231837Z/claude/workspace-path.txt",
      "evidence": "infrastructure-failures/claude-wrapper-ordering-failure.json"
    },
    {
      "contestant": "claude",
      "stage": "authentication before model invocation",
      "observed_exit_code": 1,
      "cause": "Claude Code reported Not logged in",
      "model_invocation_occurred": false,
      "usage": {
        "input_tokens": 0,
        "output_tokens": 0
      },
      "rerun_valid": true,
      "preserved_evidence": "infrastructure-failures/claude-not-authenticated/",
      "evidence": "infrastructure-failures/claude-not-authenticated/failure.json"
    },
    {
      "contestant": "codex",
      "stage": "post-invocation browser grading",
      "cause": "The grader waited indefinitely for decode() on below-fold lazy images; the contestant invocation had already completed successfully.",
      "model_invocation_occurred": true,
      "contestant_rerun": false,
      "grader_rerun_valid": true,
      "preserved_evidence": "infrastructure-failures/codex-grader-lazy-image-hang/",
      "evidence": "infrastructure-failures/codex-grader-lazy-image-hang/failure.json"
    },
    {
      "contestant": "codex",
      "stage": "post-invocation browser grading",
      "cause": "The first completed grader looked for the meta-experiment wording instead of the underlying case-study question required from contestant pages.",
      "model_invocation_occurred": true,
      "contestant_rerun": false,
      "grader_rerun_valid": true,
      "evidence": "infrastructure-failures/codex-grader-question-heuristic/failure.json"
    }
  ],
  "limitations": [
    "This evaluates one implementation pass per contestant on one source-evidence package; it does not establish general editorial ability.",
    "The qualitative judgment is an evidence-grounded editorial review, not a timed user study or blinded panel.",
    "Both final pages passed the same mechanical browser grader; that grader does not measure comprehension or visual quality.",
    "Claude Code was configured with the opus alias at high effort, but its structured output confirmed claude-opus-4-7 (plus auxiliary claude-haiku-4-5-20251001 usage), not the requested Opus 5 label. The contestant was not rerun because it had already completed a valid implementation invocation.",
    "Codex was configured explicitly as gpt-5.6-sol at high reasoning effort; the CLI did not expose a runtime model identifier, so configuration and runtime confirmation remain separate facts.",
    "The Codex browser grader required infrastructure-only reruns after a lazy-image hang and an incorrect question heuristic. Codex’s implementation invocation was not rerun.",
    "Both contestants ran sequentially under the same macOS user, not separate OS accounts; each received a clean isolated worktree and no opposing output path."
  ]
}