{
 "schema_version": "1.1",
 "tool": {
  "slug": "parsimony",
  "name": "Parsimony",
  "url": "https://parsimony.lasu.dev/",
  "status": "tested",
  "summary": "Open-source benchmark of how much code AI coding agents add",
  "category": "web app (AI benchmark)",
  "signup_required": false,
  "captcha": false,
  "bot_blocking": false,
  "api_or_mcp": false,
  "agent_completable": "full",
  "quality": 4,
  "quality_scale": 5,
  "works": "Retest 2026-10-17 ~07:07 PT after a changelog entry (DeepSeek V4.1 Flash on Live Go). No captcha/bot/signup wall on /live-go, / or /go; 0 console errors on /live-go and /go. SWE-bench Live Go board /live-go: '7 models on 138 SWE-bench Live MultiLang tasks.', 7 ranked rows sorted by 'Vs. others, same tasks'; new row 'DeepSeek V4.1 Flash \u00b7 TianxiCode 0.1.486' is #7 (1.24x, tied with Qwen3.8-27B), Solved 63.8% (= 88/138), Measured 82/88, normal styling. Footer 'Analyzer 0.10.0-beta, units tree-sitter-go-units-v2.' DeepSWE Go /go still 26 ranked rows, footer 'Analyzer 0.8.0-beta, units tree-sitter-go-units-v2.' Landing still '26 models on DeepSWE's 34 Python, 5 JavaScript, 35 TypeScript and 34 Go tasks, 4 attempts each.'",
  "breaks": "No functional failures. Minor display bug persists: the all-languages landing page footer still reads 'Analyzer undefined.' Model names on /live-go are not links; clicking a row opens nothing (task view via the Task dropdown not exercised). The Live Go page shows no note about the 24 tasks the changelog says were set aside unmeasured. Analyzer version still differs by board (0.10.0-beta Live Go, 0.8.0-beta DeepSWE Go). Request/Report issue and contributed-run links need a GitHub account (not exercised).",
  "last_tested": "2026-10-17T07:07-07:00",
  "tested_by": "Agbus",
  "test_method": "Retest after changelog change (Unreleased 2026-10-17 entry: DeepSeek V4.1 Flash \u00b7 TianxiCode on Live Go). HTTP fetch of CHANGELOG.md; browser agent on /live-go (row count, new row values, analyzer footer, row click, console), landing page footer and count line, /go footer and row count. No accounts, no uploads.",
  "agent_ready": "Agent-ready",
  "source_url": "https://news.ycombinator.com/item?id=50077619",
  "source_note": "Show HN 2026-10-13",
  "page_url": "https://agenttested.dev/directory/parsimony",
  "api_url": "https://agenttested.dev/api/tools/parsimony.json",
  "screenshot_url": null,
  "tool_version_tested": "Live Go board analyzer 0.10.0-beta (tree-sitter-go-units-v2); DeepSWE Go board 0.8.0-beta (unchanged); Python board 0.5.2-beta (not rechecked this run); all-languages landing page shows 'Analyzer undefined'",
  "tested_by_model": "Grok Bot (browser agent)",
  "release_claims": [
   {
    "claim": "Live Go gains a seventh configuration: DeepSeek V4.1 Flash \u00b7 TianxiCode 0.1.486",
    "status": "verified",
    "source": "CHANGELOG.md 2026-10-17; /live-go shows 7 models with this row ranked #7"
   },
   {
    "claim": "DeepSeek V4.1 Flash \u00b7 TianxiCode solved 88/138",
    "status": "verified",
    "source": "/live-go Solved column 63.8% (= 88/138); Measured 82/88"
   },
   {
    "claim": "Measured with the same 0.10.0-beta analyzer as the other six",
    "status": "verified",
    "source": "/live-go footer 'Analyzer 0.10.0-beta, units tree-sitter-go-units-v2.' (board-level only; no per-row analyzer shown)"
   },
   {
    "claim": "24 tasks whose patch.diff differs from preds.json are set aside unmeasured; the other six runs are unchanged",
    "status": "unverified",
    "source": "CHANGELOG.md; the page shows no set-aside note and earlier per-row values weren't recorded to compare"
   },
   {
    "claim": "Contributed runs (via CONTRIBUTING.md) show faded and unranked, named 'Model (effort) \u00b7 @you'",
    "status": "unverified",
    "source": "carried from 2026-10-16; not rechecked"
   },
   {
    "claim": "Custom benchmark JSON stays in the browser",
    "status": "unverified",
    "source": "docs claim; upload not exercised"
   }
  ],
  "history": [
   {
    "tested": "2026-10-16T15:06-07:00",
    "signup_required": false,
    "captcha": false,
    "bot_blocking": false,
    "agent_completable": "full",
    "api_or_mcp": false,
    "quality": 4,
    "works": "Retest 2026-10-16 ~15:06 PT: https://parsimony.lasu.dev/ loads HTTP 200 with no captcha/bot/signup wall; no console errors or warnings on /, /python or /go. The default landing page is now an all-languages summary ('26 models on DeepSWE's 34 Python, 5 JavaScript, 35 TypeScript and 34 Go tasks, 4 attempts each.'). DeepSWE Python board /python: '26 models on 34 DeepSWE tasks (136 attempts per model). Only 34 tasks, so the order is rough.', footer 'Analyzer 0.5.2-beta'. New feedback link 'Have spare API credits? Run a model on the tasks' opens GitHub CONTRIBUTING.md#contributed-runs (HTTP 200, 'Contributed runs' section present). 'Change chart' toggles work: 'Show other harnesses' sets ?harnesses=all, 'Show my runs' off adds &selfrun=0; ranked table stays 26 rows. DeepSWE Go /go: 26 ranked rows, 'Analyzer 0.8.0-beta, units tree-sitter-go-units-v2'.",
    "breaks": "No functional failures. Minor display bug: the all-languages landing page footer reads 'Analyzer undefined.' (the footer template uses data.analyzer_version, which the combined page doesn't set). The /go count line reuses the Python wording ('26 models on 34 DeepSWE tasks (136 attempts per model)'). Toggles are hidden until 'Change chart' is expanded. Analyzer version still differs by board. Request/Report issue and contributed-run links need a GitHub account to act on (not exercised).",
    "tool_version_tested": "Python board analyzer 0.5.2-beta; Go board analyzer 0.8.0-beta, units tree-sitter-go-units-v2 (unchanged); all-languages landing page shows no analyzer version",
    "tested_by_model": "Grok Bot (browser agent)",
    "test_method": "Retest after page_fingerprint text change (new 'Have spare API credits? Run a model on the tasks' link; assets unchanged). Browser agent: landing page, Python board count and footer, new contributing link, both chart toggles with URL checks, Go board, console listeners; plus HTTP fetch of the page and CONTRIBUTING.md. No accounts; no JSON upload.",
    "release_claims": [
     {
      "claim": "Contributed runs (via CONTRIBUTING.md) show faded and unranked, named 'Model (effort) \u00b7 @you'",
      "status": "unverified",
      "source": "CONTRIBUTING.md 'Contributed runs' section exists and the site links to it; no contributed-run rows were identified on the boards this retest"
     },
     {
      "claim": "Custom benchmark JSON stays in the browser",
      "status": "unverified",
      "source": "docs claim; upload not exercised this retest"
     }
    ]
   },
   {
    "tested": "2026-10-16T14:07-07:00",
    "signup_required": false,
    "captcha": false,
    "bot_blocking": false,
    "agent_completable": "full",
    "api_or_mcp": false,
    "quality": 4,
    "works": "Retest 2026-10-16 ~14:07 PT: loads HTTP 200 with no captcha/bot/signup wall; console 0 errors, 0 warnings. DeepSWE Python board: 26 ranked models ('26 models on 34 DeepSWE tasks (136 attempts per model). Only 34 tasks, so the order is rough.'), footer 'Analyzer 0.5.2-beta', plus 8 unranked 'My runs' rows. Toggles (inside the collapsed 'Change chart' disclosure) work: 'Show other harnesses' sets /?harnesses=all and adds 4 pi rows to My runs (ranked 26 unchanged); 'Show my runs' off sets &selfrun=0 and leaves only the 26 ranked rows. DeepSWE Go /go: 26 ranked rows, 'Analyzer 0.8.0-beta, units tree-sitter-go-units-v2'.",
    "breaks": "No functional failures. Trigger was methodology copy only: the Limits section now adds 'A model whose fixes mostly delete code can rank first with a negative net, and failed attempts that change nothing pull the all-measured mean toward zero.' The My-runs help text changed from 'first 5\u201310' to 'first few' in the static HTML; the rendered page reads 'first 3\u201310 DeepSWE Python tasks'. Toggles are hidden until 'Change chart' is expanded. Chart labels are crowded; on Go, Gemini 3.1 Pro sits off the plot (page says so). Analyzer version still differs by board. Request/Report issue links need a GitHub account (not exercised).",
    "tool_version_tested": "Python board analyzer 0.5.2-beta; Go board analyzer 0.8.0-beta, units tree-sitter-go-units-v2 (unchanged)",
    "tested_by_model": "Grok Bot (browser agent)",
    "test_method": "Retest after page_fingerprint text change (methodology wording; assets unchanged). Browser agent: Python leaderboard count, both toggles with URL checks, Go board, console check; no accounts; no JSON upload.",
    "release_claims": [
     {
      "claim": "Custom benchmark JSON stays in the browser",
      "status": "unverified",
      "source": "docs claim; upload not exercised this retest"
     }
    ]
   },
   {
    "tested": "2026-10-16T08:05-07:00",
    "signup_required": false,
    "captcha": false,
    "bot_blocking": false,
    "agent_completable": "full",
    "api_or_mcp": false,
    "quality": 4,
    "works": "Retest 2026-10-16 ~08:05 PT: https://parsimony.lasu.dev/ loads HTTP 200 with no captcha/bot/signup wall. DeepSWE Python board: 26 ranked models; footer analyzer 0.5.2-beta. DeepSWE Go /go: 26 ranked; analyzer 0.8.0-beta, units tree-sitter-go-units-v2; page notes 'Gemini 3.1 Pro is off the scale'. Fingerprint trigger copy now explains coding units (net units = added \u2212 removed). My-runs rows (8) visible with self-run-toggle checked in DOM. No JSON upload exercised.",
    "breaks": "No functional failures vs prior full/4. Toggle labels start with HTML hidden attr (JS should reveal after self-run data loads); this run's browser agent did not successfully click toggles (labels reported hidden) \u2014 toggles not re-exercised this run, not scored as a regression. No Request/Report issue exercised. Analyzer version still differs by board.",
    "tool_version_tested": "Python board analyzer 0.5.2-beta; Go board analyzer 0.8.0-beta (unchanged)",
    "tested_by_model": "Grok Bot (browser agent)",
    "test_method": "Retest after page_fingerprint coding-units wording change (counts coding units, syntax elements like calls, names and literals rather than lines; net units = added \u2212 removed). Browser agent: Python leaderboard + Go board; toggles not re-clicked this run. No accounts; no JSON upload.",
    "release_claims": [
     {
      "claim": "Custom benchmark JSON stays in the browser",
      "status": "unverified",
      "source": "docs claim; upload not exercised this retest (nor prior 2026-10-16 07:04)"
     }
    ]
   },
   {
    "tested": "2026-10-16T07:04-07:00",
    "signup_required": false,
    "captcha": false,
    "bot_blocking": false,
    "agent_completable": "full",
    "api_or_mcp": false,
    "quality": 4,
    "works": "Retest 2026-10-16: leaderboard loads (200) with no captcha/bot/signup wall and no console errors or failed requests. DeepSWE Python board: 26 ranked models ('26 models on 34 DeepSWE tasks (136 attempts per model)'). 'Show other harnesses' sets ?harnesses=all and raises my-runs rows 8 to 12 (ranked stays 26). 'Show my runs' is on by default; unchecking sets &selfrun=0 and hides those rows. DeepSWE Go board routes to /go with 26 ranked rows. Open-benchmark-JSON picker not exercised.",
    "breaks": "No functional failures. The new 'Swipe the table sideways for more columns.' hint is in the DOM but hidden at desktop width (mobile hint; it was the fingerprint trigger). 'Show my runs' means maintainer runs. Analyzer version differs by board. Request/Report issue links need a GitHub account (not exercised).",
    "tool_version_tested": "Python board analyzer 0.5.2-beta; Go board analyzer 0.8.0-beta (unchanged)",
    "tested_by_model": "Grok Bot (browser agent)",
    "test_method": "Retest after page_fingerprint text change (new mobile swipe hint; assets unchanged). Browser agent: leaderboard count, harness and my-runs toggles, Go board switch, console check; no accounts; no JSON upload.",
    "release_claims": []
   },
   {
    "tested": "2026-10-15T07:35-07:00",
    "signup_required": false,
    "captcha": false,
    "bot_blocking": false,
    "agent_completable": "full",
    "api_or_mcp": false,
    "quality": 4,
    "works": "Homepage/leaderboard loads with no captcha/bot/signup wall. Full benchmark JSON embedded as #parsimony-data (~497KB) plus tables in HTML. DeepSWE Python board: 26 ranked models with scores/solve rates readable as text. Board nav (/verified, /go, etc.) works. 'Show other harnesses' toggles URL ?harnesses=all and adds My-runs rows; 'Show my runs' toggles maintainer runs section. Open-benchmark-JSON file picker present with no account (files stay in browser per docs; upload not exercised).",
    "breaks": "No functional failures. 'Show my runs' means maintainer runs, not visitor runs. 'Measured' column lacks on-page definition. Analyzer version differs by board (Python 0.5.2-beta vs Go 0.8.0-beta). Example JSON unit labels vs method text are ambiguous. Page notes small task count so order can be rough. Request/Report issue links need a GitHub account to submit (not exercised).",
    "tool_version_tested": "Python board: analyzer 0.5.2-beta, score_version parsimony-80-20-v0.5, panel deepswe-v1.1-python-26-pooled-80-20-v0.5; Go board analyzer 0.8.0-beta. GitHub latest release deepswe-go-v0.8 (pre-release).",
    "tested_by_model": "Grok Bot (browser agent)",
    "test_method": "First test after page_fingerprint text change. Browser agent: leaderboard read, harness/my-runs toggles, board switch; no accounts; no JSON upload.",
    "release_claims": [
     {
      "claim": "DeepSWE Go v2 release: analyzer 0.8.0-beta, 26 configs \u00d7 34 tasks, research preview not an official ranking",
      "status": "verified",
      "source": "GitHub release deepswe-go-v0.8 copy matched site Go board analyzer 0.8.0-beta / 26 rows; asset checksums not verified"
     },
     {
      "claim": "Custom benchmark JSON stays in the browser",
      "status": "unverified",
      "source": "docs claim; upload not exercised"
     }
    ]
   }
  ],
  "scoring_note": null,
  "security_flags": []
 }
}