<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "journalpublishing.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="2.0" xml:lang="en" article-type="research-article"><front><journal-meta><journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id><journal-id journal-id-type="publisher-id">jmir</journal-id><journal-id journal-id-type="index">1</journal-id><journal-title>Journal of Medical Internet Research</journal-title><abbrev-journal-title>J Med Internet Res</abbrev-journal-title><issn pub-type="epub">1438-8871</issn><publisher><publisher-name>JMIR Publications</publisher-name><publisher-loc>Toronto, Canada</publisher-loc></publisher></journal-meta><article-meta><article-id pub-id-type="publisher-id">v28i1e96885</article-id><article-id pub-id-type="doi">10.2196/96885</article-id><article-categories><subj-group subj-group-type="heading"><subject>Original Paper</subject></subj-group></article-categories><title-group><article-title>Medical Reasoning in Large Language Models: Development and Evaluation of the MedEvalArena Benchmark</article-title></title-group><contrib-group><contrib contrib-type="author"><name name-style="western"><surname>Prem</surname><given-names>Preethi</given-names></name><degrees>BS</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Kuppa</surname><given-names>Vikasini</given-names></name><xref ref-type="aff" rid="aff2">2</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Shidara</surname><given-names>Kie</given-names></name><degrees>BS</degrees><xref ref-type="aff" rid="aff3">3</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Wheeler</surname><given-names>Esm&#x00E9;</given-names></name><degrees>BA</degrees><xref ref-type="aff" rid="aff3">3</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Chang</surname><given-names>Gene</given-names></name><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Liu</surname><given-names>Feng</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="aff5">5</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Alaa</surname><given-names>Ahmed</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author" corresp="yes"><name name-style="western"><surname>Bernardo</surname><given-names>Danilo</given-names></name><degrees>MD</degrees><xref ref-type="aff" rid="aff3">3</xref></contrib></contrib-group><aff id="aff1"><institution>Carle Illinois College of Medicine, University of Illinois Urbana-Champaign</institution><addr-line>Champaign</addr-line><addr-line>IL</addr-line><country>United States</country></aff><aff id="aff2"><institution>College of Natural and Agricultural Sciences, University of California, Riverside</institution><addr-line>Riverside</addr-line><addr-line>CA</addr-line><country>United States</country></aff><aff id="aff3"><institution>Weill Institute of Neurology and Neurosciences, University of California, San Francisco</institution><addr-line>1825 Fourth Street, Suite 5A</addr-line><addr-line>San Francisco</addr-line><addr-line>CA</addr-line><country>United States</country></aff><aff id="aff4"><institution>Department of Neurosciences, University of California, Berkeley</institution><addr-line>Berkeley</addr-line><addr-line>CA</addr-line><country>United States</country></aff><aff id="aff5"><institution>Department of Industrial &#x0026; Systems Engineering, Rutgers University</institution><addr-line>Piscataway</addr-line><addr-line>NJ</addr-line><country>United States</country></aff><contrib-group><contrib contrib-type="editor"><name name-style="western"><surname>Coristine</surname><given-names>Andrew</given-names></name></contrib></contrib-group><contrib-group><contrib contrib-type="reviewer"><name name-style="western"><surname>Chen</surname><given-names>Mei</given-names></name></contrib><contrib contrib-type="reviewer"><name name-style="western"><surname>Koike</surname><given-names>Takeshi</given-names></name></contrib><contrib contrib-type="reviewer"><name name-style="western"><surname>Hu</surname><given-names>Yihan</given-names></name></contrib><contrib contrib-type="reviewer"><name name-style="western"><surname>Liu</surname><given-names>Zhao</given-names></name></contrib></contrib-group><author-notes><corresp>Correspondence to Danilo Bernardo, MD, Weill Institute of Neurology and Neurosciences, University of California, San Francisco, 1825 Fourth Street, Suite 5A, San Francisco, CA, 94143, United States, 1 415-353-2437; <email>dbernardoj@gmail.com</email></corresp></author-notes><pub-date pub-type="collection"><year>2026</year></pub-date><pub-date pub-type="epub"><day>20</day><month>11</month><year>2026</year></pub-date><volume>28</volume><elocation-id>e96885</elocation-id><history><date date-type="received"><day>01</day><month>04</month><year>2026</year></date><date date-type="rev-request"><day>02</day><month>06</month><year>2026</year></date><date date-type="rev-recd"><day>15</day><month>10</month><year>2026</year></date><date date-type="accepted"><day>20</day><month>10</month><year>2026</year></date></history><copyright-statement>&#x00A9; Preethi Prem, Vikasini Kuppa, Kie Shidara, Esm&#x00E9; Wheeler, Gene Chang, Feng Liu, Ahmed Alaa, Danilo Bernardo. Originally published in the Journal of Medical Internet Research (<ext-link ext-link-type="uri" xlink:href="https://www.jmir.org">https://www.jmir.org</ext-link>), 20.11.2026. </copyright-statement><copyright-year>2026</copyright-year><license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (<ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by/4.0/">https://creativecommons.org/licenses/by/4.0/</ext-link>), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research (ISSN 1438-8871), is properly cited. The complete bibliographic information, a link to the original publication on <ext-link ext-link-type="uri" xlink:href="https://www.jmir.org/">https://www.jmir.org/</ext-link>, as well as this copyright and license information must be included.</p></license><self-uri xlink:type="simple" xlink:href="https://www.jmir.org/2026/1/e96885"/><abstract><sec><title>Background</title><p>Large language models (LLMs) perform strongly on medical specialty board multiple-choice questions (MCQs) but underperform in complex medical reasoning. This gap indicates a need to improve LLM medical reasoning and evaluation paradigms.</p></sec><sec><title>Objective</title><p>This study aimed to develop and evaluate MedEvalArena, a dynamically regenerable benchmark for assessing medical reasoning in LLMs.</p></sec><sec sec-type="methods"><title>Methods</title><p>In a symmetric round-robin format, 6 frontier LLMs from the OpenAI, Grok, Gemini, Claude, Kimi, and DeepSeek families generated challenging board-style medical MCQs, adjudicated question validity on an ensemble LLM-as-judge panel, and answered the validated examination. Candidate items were generated until each model contributed 50 validated questions according to Association of American Medical Colleges (AAMC)/American Board of Medical Specialties (ABMS) physician specialty distributions, yielding a 300-question examination. Judges adjudicated candidate questions&#x2019; logical validity and medical accuracy. We compared question-generation validity rates, assessed leave-one-out judge agreement, and tested for self-judging preference. Questions were audited for overlap with public MedQA (medical question-answering) benchmarks. We tested question-answering (QA) accuracy across LLMs using a binomial-logit generalized linear mixed model likelihood-ratio test. A generator-stratified 90-question subset was completed by 5 board-certified physicians for paired human-LLM analysis. We evaluated inference cost-accuracy tradeoffs. Multiple comparisons were adjusted by the Benjamini-Hochberg (BH) procedure.</p></sec><sec sec-type="results"><title>Results</title><p>Judge-panel validity rates were significantly higher for questions generated by OpenAI (94.8%, Wilson 95% CI 85.9%-98.2%), Gemini (92.3%, 95% CI 83.2%-96.7%), and Claude (83.3%, 95% CI 72.6%-90.4%) models than those generated by Kimi (63.8%, 95% CI 54.3%-72.4%), Grok (56%, 95% CI 45.8%-65.8%), and DeepSeek (46.0%, 95% CI 37.4%-54.7%). Kimi also demonstrated significantly higher validity than DeepSeek (2-sample tests of proportions; BH-adjusted <italic>P</italic>=.01). The public-source similarity audit found no exact or near-duplicate matches. Leave-one-out judge agreement was moderate. Generator-specific same-family judging coefficients were nonnegative (0 to 11.0 percentage points); however, none remained significant after BH adjustment. There was no significant difference between the 6 frontier LLM QA accuracies, which ranged from 85.7% (257/300) to 91.7% (275/300), on the 300-question examination (generalized linear mixed model likelihood-ratio test <italic>P</italic>=.07). The study had 80% power to detect LLM accuracy differences of 9.2 percentage points; nonsignificance did not establish equivalence. On the 90-question subset, physician accuracy was 85.6% (95% CI 79.4%&#x2010;91.7%; stratified normal with finite-population correction), compared with aggregate LLM performance of 94.4% (85/90; Wilson 95% CI 87.6%&#x2010;97.6%). This difference was not significant (McNemar test, <italic>P</italic>=.08); however, the study was powered to detect a human vs LLM accuracy difference of 14% to 15%. Answering-only costs varied by nearly 2 orders of magnitude, with multiple models on the cost-accuracy Pareto frontier.</p></sec><sec sec-type="conclusions"><title>Conclusions</title><p>MedEvalArena enables scalable, regenerable, peer-judged benchmarking of LLM medical reasoning. Question-generation validity differentiated model families more clearly than answering accuracy, suggesting that valid MCQ generation may complement QA benchmarks for assessing LLM medical reasoning.</p></sec></abstract><kwd-group><kwd>medical reasoning</kwd><kwd>large language model</kwd><kwd>LLM</kwd><kwd>AI</kwd><kwd>reasoning</kwd><kwd>benchmark</kwd></kwd-group></article-meta></front><body><sec id="s1" sec-type="intro"><title>Introduction</title><p>The performance of large language models (LLMs) in medical reasoning tasks has quickly progressed [<xref ref-type="bibr" rid="ref1">1</xref>], now matching or exceeding human physician performance on standardized medical examinations, such as the United States Medical Licensing Examination (USMLE) step examinations [<xref ref-type="bibr" rid="ref2">2</xref>-<xref ref-type="bibr" rid="ref4">4</xref>]. However, as LLMs continue to improve, their performance remains limited on medical benchmarks requiring open-ended reasoning and future planning [<xref ref-type="bibr" rid="ref5">5</xref>-<xref ref-type="bibr" rid="ref9">9</xref>], demonstrating the need for more robust LLM medical reasoning and evaluation frameworks.</p><p>Conventional medical reasoning evaluation relies on benchmarks that require academic peer review and validation [<xref ref-type="bibr" rid="ref10">10</xref>]. However, LLM reasoning capabilities are improving rapidly, driving a continuous need for novel and more challenging benchmarks of reasoning capabilities [<xref ref-type="bibr" rid="ref11">11</xref>]. This introduces a significant lag between the abilities of frontier LLMs and our understanding of their medical reasoning competence. Addressing these limitations motivates dynamic evaluation frameworks that are able to adapt alongside LLMs, specifically evaluation pipelines that enable rapid benchmark reinstantiation to facilitate tighter feedback-loops for LLM improvement.</p><p>To mitigate benchmark saturation and dataset contamination while keeping pace with rapid model development, recent work has introduced LLM benchmarks that can be updated with human guidance. Approaches such as DynaBench [<xref ref-type="bibr" rid="ref12">12</xref>] and LiveBench [<xref ref-type="bibr" rid="ref13">13</xref>] are capable of intermittently updating assessments, with human oversight of task curation. BenchBuilder operationalizes updating at scale via automatic generation of a sizable, diverse benchmark of tasks at low cost [<xref ref-type="bibr" rid="ref14">14</xref>]. More recently, Autobench incorporates the LLM-as-judge paradigm into dynamic benchmark generation, using a round-robin framework in which models generate questions, complete questions, and evaluate outputs via peer assessment [<xref ref-type="bibr" rid="ref15">15</xref>].</p><p>Motivated by the potential of these approaches to assess and advance LLM performance in medical reasoning, we present MedEvalArena, a dynamic framework for evaluating the medical reasoning capabilities of LLMs through peer assessment. In this approach, LLMs participate in a symmetric round-robin design in which each model generates challenging medical questions designed to assess other models&#x2019; reasoning capabilities. We use an LLM-as-judge ensemble paradigm to ensure the validity of LLM-generated questions across 2 dimensions: logical coherence and medical accuracy [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref17">17</xref>]. All models are then evaluated on the complete question set, enabling scalable benchmarking of LLM medical reasoning capabilities.</p></sec><sec id="s2" sec-type="methods"><title>Methods</title><p>We construct our MedEvalArena in 3 stages (<xref ref-type="fig" rid="figure1">Figure 1</xref>): (1) question generation by LLMs acting as question generators, (2) LLMs serving in an LLM-as-judge ensemble that assesses item validity, and (3) a round-robin evaluation in which multiple LLMs take all questions as examinees.</p><fig position="float" id="figure1"><label>Figure 1.</label><caption><p>Round-robin large language model (LLM) evaluation scheme. Each of the 6 LLM families first generates questions, while the LLM-as-judge ensemble (generation and validation phases), consisting of the same 6 judges, adjudicates the validity of the generated questions. The resulting set of valid questions is then used as a shared benchmark: every model answers every question, and aggregate performance across the entire examination is used to compare models. MCQ: multiple-choice question.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig01.png"/></fig><sec id="s2-1"><title>Model Selection</title><p>We included LLMs for examination generation and LLM-as-judge that were the leading models from the top 6 frontier labs, per the artificialanalysis.ai rankings, with a model cutoff date of November 20, 2025 [<xref ref-type="bibr" rid="ref18">18</xref>]. This resulted in 6 examination generators, including gpt-5.1-2025-11-13 (OpenAI [<xref ref-type="bibr" rid="ref19">19</xref>]), claude-opus-4-5-20251101 (Anthropic [<xref ref-type="bibr" rid="ref20">20</xref>]), gemini-3-pro-preview (Google [<xref ref-type="bibr" rid="ref21">21</xref>]), deepseek-v3.2 [<xref ref-type="bibr" rid="ref22">22</xref>], grok-4-0709 (xAI [<xref ref-type="bibr" rid="ref23">23</xref>]), and Kimi kimi-k2-thinking (Moonshot AI [<xref ref-type="bibr" rid="ref24">24</xref>]). These 6 models subsequently also served in the LLM-as-judge ensemble. The same 6 model families served as question generators, judges, and primary answer models. The subsequent cost analysis included additional lower-cost or weaker-reasoning answer models from the same broad vendor families: grok-4&#x2010;1-fast-reasoning, claude-haiku-4-5-20251001, gemini-2.5-flash-lite, and gpt-5-nano-2025-08-07. The complete model roster is provided in Table S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p></sec><sec id="s2-2"><title>Stage 1: Round-Robin Question Generation</title><p>In the first stage, each generator model <italic>g</italic> generates board-level medical multiple-choice questions (MCQs). For each <italic>g</italic>, we specified a total of 50 questions with a topic distribution determined by the specialty representation of the US physician workforce as published by the Association of American Medical Colleges (AAMC) [<xref ref-type="bibr" rid="ref25">25</xref>]. This provides a population-based representation of generalist core questions from internal medicine, family medicine, and surgery, as well as other specialty and subspecialty board questions. Specialty names were harmonized to match the American Board of Medical Specialties (ABMS) specialty categories [<xref ref-type="bibr" rid="ref26">26</xref>]. The American Board of Family Medicine (ABFM) and American Board of Internal Medicine (ABIM) topic distribution blueprints were used only to define generalist family medicine, internal medicine, and medical subspecialty topic distributions [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. Surgical subspecialties such as neurological surgery and vascular surgery were included because they appear in the overarching AAMC and ABMS specialty mappings. The topic distribution is provided in Table S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. The generator is instructed, via a structured system prompt, to produce extremely difficult medical board&#x2013;style questions with exactly 1 best answer and an accompanying explanation for that answer. Each LLM model is queried once per question with this specification, using a low sampling temperature (<italic>T</italic>=0) to favor deterministic reproducibility across benchmark instantiations. This reduces stochastic variation but does not exclude memorized or benchmark-like outputs.</p></sec><sec id="s2-3"><title>Stage 2: Ensemble LLM-as-Judge Question Validation</title><p>In the second stage, to filter out clinically inaccurate or logically incoherent items, we use an ensemble LLM-as-judge panel that reviews each question-answer pair. The judge ensemble receives, for every question, the question stem, the full list of answer options, the designated correct answer, and the explanation produced by the generator, together with a detailed rubric describing the desired evaluation criteria. For each question, each judge in the judge ensemble is prompted to return a structured assessment in which it provides (1) a free-text analytic rationale, (2) an integer medical accuracy score on a 5-point ordinal scale, (3) a binary indicator of logical validity specifying whether the item is well-posed and has a defensible unique answer, and (4) a categorical label that summarizes the primary failure mode when logical validity does not hold, specifically one of the following: contradiction, miskeyed answer, multiple defensible answers, or underspecification. The medical accuracy score is rated on the following scale: 1=dangerous/false: clear hallucinations or advice that plausibly causes harm; strongly contradicts standard care. 2=inaccurate: major medical errors or clearly outdated guidance that would mislead a learner. 3=minor issues: mostly correct but missing important details, uses nonstandard terminology, or is clinically unrealistic in a noncritical way. 4=accurate: consistent with current standard care/guidelines; clinically sensible. 5=gold standard: fully accurate, precise, realistic, and board-relevant.</p><p>For questions to be included in the subsequent primary outcome evaluation phase, a question is kept only if a strict majority of judges rate it logically valid and a strict majority of judges assign a medical-accuracy score (<italic>&#x03C4;</italic>) of <italic>&#x03C4;</italic>&#x2265;3. Specifically, we define a strict majority of judges to be at least 4 of 6 judges. Thus, 3 to 3 decisions fail the strict-majority criterion and are excluded. Question generators repeated stages 1 and 2 until the specified AAMC/ABMS distribution of 50 valid questions per examination was met. Thus, generators with lower validity rates resulted in more rejected candidate attempts. The rule used to select among surplus passed items was sequential by generation order.</p></sec><sec id="s2-4"><title>Stage 3: Round-Robin Examination Evaluation</title><p>We evaluate each model <italic>a</italic> on all questions produced by each generator model <italic>g</italic>. For each question, the taker is given the stem and 5 answer options and instructed to output exactly 1 option label as their final answer. All taker queries use <italic>T</italic>=0 for reproducibility. For each (<italic>g,a</italic>) pair, we compute accuracy as the proportion of questions answered correctly.</p></sec><sec id="s2-5"><title>Question Similarity Audit</title><p>After constructing the final examination, we performed a public-source similarity audit of the 300 selected questions against MedQA (medical question-answering), MedMCQA (medical multiple-choice question-answering), PubMedQA (PubMed question-answering), MMLU (Massive Multitask Language Understanding) medical subsets, and the official public USMLE step 1 and step 2 clinical knowledge sample-question pages [<xref ref-type="bibr" rid="ref29">29</xref>-<xref ref-type="bibr" rid="ref34">34</xref>]. The audit used normalized exact matching and 10-token shingle overlap/containment on question stems plus answer options.</p></sec><sec id="s2-6"><title>Question-Generation and Question-Answering Inference Cost Analysis</title><p>Question-generation inference costs were obtained using costs incurred for the question-generation queries for all questions generated per model in order to reach the prespecified AAMC/ABMS topic distribution for the final 300-question examination. Questions were generated using batch processing as made available through the respective LLM APIs. Surplus validated questions occurred because questions were generated in batches and pass/fail status was not known until after LLM-as-judge review, requiring batch generation of more candidates than the exact remaining quota. These surplus questions, as well as all candidate questions that were rejected, are included in each model&#x2019;s respective question-generation costs. Answering-only inference costs were analyzed as the costs incurred by each respective answerer model to complete the 300-question MedEvalArena examination. For both question-generation and question-answering tasks, models were accessed between October 1, 2025, and November 30, 2025. Costs are reported in US dollars per examination. Pareto analysis was performed as an analysis of answerer cost-vs-accuracy. Since the question-answer phase occurs after the question-generation phase, the presented Pareto frontier does not directly reflect question-generation costs.</p></sec><sec id="s2-7"><title>Statistical Analysis of LLM Question Generation and Question Answering</title><p>We evaluated differences in LLM question generation validity rates with pairwise Pearson chi-square tests for equality of 2 independent proportions at the generator level, using accepted and total candidate counts. All generator pairs used the topic counts shown in Tables S3 and S4 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. We performed leave-one-out cross-validation (LOOCV) of individual judges across all questions to assess individual judge agreement with the ensemble. Each judge is held out in turn and compared against the consensus formed by the remaining 5 judges (N=5). To assess logical validity, agreement is measured as accuracy with the ensemble&#x2019;s binary pass/fail decision. For medical accuracy score, agreement is measured as exact match with the ensemble&#x2019;s median score. By using 5000 bootstrap resampling iterations, 95% CIs were generated. Self-family judging preference was explored using a judge&#x2013;item-level interaction fixed-effects linear probability sensitivity model with generator-specific same-family indicators, item and judge fixed effects, and SEs clustered by item.</p><p>We tested for differences in item-level question accuracy across LLMs. Using the R (R Foundation for Statistical Computing) lme4 package, we fit a binomial-logit generalized linear mixed model (GLMM) with LLM identity as a fixed effect, quiz identity (each respective quiz is generated by a single LLM) as a fixed effect, and random intercepts for questions within examinations [<xref ref-type="bibr" rid="ref35">35</xref>,<xref ref-type="bibr" rid="ref36">36</xref>]. The fitted formula was correct~ model + quiz_id + (1 | quiz_id:question_id). We tested the LLM effect via a likelihood-ratio comparison to a null model [<xref ref-type="bibr" rid="ref37">37</xref>]. Model optimizer diagnostics indicated that both the full and null models converged successfully. We conducted simulation-based post hoc power analysis using the same paired 300-question by 6-model response structure. Simulations preserved the observed item-difficulty distribution and model accuracy effects, used 2000 replicates per scenario with a fixed random seed, and tested model differences using the Cochran Q test for paired binary outcomes. A computationally tractable omnibus test for paired binary outcomes, the Cochran Q test, accounts for the 6 models answering the same questions while avoiding repeated fitting of the full and null GLMMs across multiple replicates per scenario. We assessed generator-answer interaction patterns from the generator-by-answer matrix to assess whether answer models appeared to perform better on questions generated by the same model. All statistical comparisons were adjusted for multiple comparisons with the Benjamini-Hochberg (BH) procedure.</p></sec><sec id="s2-8"><title>Specialty Analysis</title><p>Specialty-level performance was evaluated by grouping topics into surgical, medical, and other specialties. Surgical specialties included ABMS surgical and procedural specialties. Medical specialties included internal medicine subspecialty topics and other nonprocedural medical domains. Remaining categories were grouped as other. Mean correctness was calculated for each topic across the 300-validated questions. For the global topic test, the item was the resampling unit, and each item was scored by mean correctness across available answer models. The permutation analysis used the one-way ANOVA <italic>F</italic> statistic on item scores by topic, 5000 permutations, and quiz-stratified label shuffling to preserve generator-examination structure while allowing unequal topic sample sizes.</p><p>Exploratory pairwise comparisons among surgical, medical, and other specialty groups used differences in mean question-level correctness across the 6 answering models. Each comparison included only its 2 groups, with 2-sided <italic>P</italic> values estimated using 5000 generator-stratified permutations of group labels, preserving group sizes within each generator. <italic>P</italic> values were adjusted across the 3 comparisons using the BH procedure.</p></sec><sec id="s2-9"><title>Physician Comparison Analysis</title><p>For the human comparison, 6 board-certified physician raters were recruited from UCSF (University of California, San Francisco) Medical Center and Kolabtree. All physician raters had passed the USMLE or United Kingdom Medical Licensing Assessment (UKMLA) and had at least 5 years of clinical experience. Five physicians were assigned to the rating panel, and 1 physician was assigned to adjudicate questions that were flagged as medically inaccurate or illogical. Three had internal medicine board certification, 1 had neurology board certification, and 2 had general surgery board certification. Physician raters were blinded to the source LLM for each question. Details of the statistical comparison with LLM performance follow.</p><p>The physician comparison was designed as a descriptive human reference for contextualizing question difficulty rather than as a powered test for definitive human-LLM superiority, noninferiority, or an equivalence study. We selected a 90-question incomplete block design to limit physician rater burden. Questions were generator-stratified for equal representation across the 6 LLM question generators. To characterize the detectable effect size under the feasible 90-question design, we calibrated McNemar paired-discordance using item-level paired physician and LLM responses from a recent physician vs LLM comparison on medical MCQ reasoning [<xref ref-type="bibr" rid="ref8">8</xref>]. This study reported 5-physician mean accuracy of 0.66 (SD 0.13), DeepSeek-R1 accuracy of 0.52 (95% CI 0.515-0.534), and a paired-bootstrap <italic>P</italic>=.01. A normal paired-proportion approximation gave paired-discordance of 0.30. Under a 90-question human-evaluation design, this provided approximately 80% power at &#x03B1;=.05 to detect a paired physician-vs-LLM accuracy difference of 14 to 15 percentage points (PPs), comparable to the 14-point physician-vs-DeepSeek-R1 difference reported in Medical Abstraction and Reasoning Corpus Question and Answer (mARC-QA). Thus, the physician comparison was not powered to exclude smaller differences or establish equivalence.</p><p>The 90-question comparison set was randomly sampled from the validated question pool using 6 generator strata, one for each LLM generator. Each generator stratum contributed 15 sampled questions, including 10 double-rated questions and 5 single-rated questions. Each of the 5 raters completed 5 assignments per generator stratum, consisting of 4 shared double-rated assignments and 1 unique single-rated assignment. For the paired human vs LLM comparison, we compared physician correctness with aggregate LLM correctness on the same 90-question set. The aggregate LLM outcome was coded as correct when at least 3 of the 6 LLMs answered the question correctly. For physician responses, single-rated items used the sole physician&#x2019;s correctness label. Double-rated items used the agreed correctness label when the 2 physicians agreed. For questions where the 2 physician raters disagreed, a third physician provided adjudication by providing an additional label, thereby ensuring majority agreement (2 out of 3 physicians). This physician was a member of the 5-physician panel who had not originally answered the question being adjudicated and was blinded to the original raters&#x2019; answers. An exact test for paired binary outcomes, developed by McNemar, was applied.</p><p>Physician raters were also instructed to flag questions they considered potentially medically inaccurate or illogical. Any question flagged by at least 1 rater underwent adjudication by an additional physician, who was not part of the 5-physician panel and who had access to OpenEvidence.</p><p>All CIs for the physician-LLM comparison were 2-sided at 95%. The overall physician CI used a generator-stratified normal approximation with finite-population correction, accounting for sampling 15 questions without replacement from each of 6 generator strata containing 50 questions. The interval was calculated as the stratum-weighted accuracy estimate &#x00B1;1.96 times its SE. Generator-specific physician CIs used corresponding within-stratum normal approximations with finite-population correction, with limits restricted to 0% to 100%. These intervals reflect item-sampling uncertainty within the fixed 300-question pool, conditional on the physician panel and rating protocol. Descriptive single-rated and double-rated physician intervals and the aggregate LLM interval used the Wilson score method without continuity correction. No bootstrap resampling was used for these comparison intervals.</p></sec><sec id="s2-10"><title>Sensitivity Analysis</title><p>We conducted a secondary sensitivity analysis by varying the minimum medical accuracy threshold (<italic>&#x03C4;</italic>) required for examination inclusion. We evaluated 4 criteria: a majority of the panel of judges assigning a rating of at least <italic>&#x03C4;</italic>=3, <italic>&#x03C4;</italic>=4, or <italic>&#x03C4;</italic>=5, and the unanimous assignment of a rating of <italic>&#x03C4;</italic>=5. For each threshold, we recalculated the number of available questions per generator and answer-model accuracy on the retained items. We used a linear probability model to test whether a judge was more likely to assign a medical-accuracy score of 5 to questions that the same model answered correctly. The model included question and judge fixed effects, an indicator for whether the judge had generated the question, and a separate answer-correctness coefficient for each of the 6 models. Uncertainty was estimated using 10,000 generator-stratified bootstrap samples.</p></sec><sec id="s2-11"><title>Ethical Considerations</title><p>This study did not involve patient contact or identifiable patient data. The physician rater component was reviewed under the current applicable UCSF Institutional Review Board (IRB) protocol number 24&#x2010;42911. Physician responses were analyzed in deidentified, aggregate form. Physician raters were recruited from UCSF Medical Center and Kolabtree and received monetary compensation at the market rate for rater time; compensation was not tied to answer choices or performance.</p></sec></sec><sec id="s3" sec-type="results"><title>Results</title><sec id="s3-1"><title>Dataset Characteristics</title><p>Each examination generator was used to create a single 50-question examination per generator, following a topic distribution weighted by the US physician specialty distribution reported by the AAMC/ABMS [<xref ref-type="bibr" rid="ref25">25</xref>,<xref ref-type="bibr" rid="ref26">26</xref>]. This yielded a total of 300 questions containing both generalist and specialty board questions. The topic distribution is shown in Table S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p></sec><sec id="s3-2"><title>An LLM-as-Judge Evaluation of MCQ Validity</title><p>To assess question validity, we evaluated each generated MCQ under an LLM-as-judge paradigm using strict rating criteria for medical accuracy and logical coherence. A question was deemed valid only if the strict majority of judges rated the question as logically coherent and the majority of LLM judges rated the question as medically accurate with a score greater than or equal to 3. Across all models, gpt-5.1 achieved the highest proportion of generated questions passing as valid, at 94.8% (55/58), and deepseek-v3.2 had the lowest proportion passing as valid, at 46.0% (57/124; <xref ref-type="table" rid="table1">Table 1</xref>). The total numbers of generated candidates and those that passed validation are shown in Table S3 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. Pairwise 2-sample proportion tests indicate no detectable difference among GPT-5.1, Gemini, and Claude, whereas these models outperformed the DeepSeek, Grok, and Kimi models with <italic>P</italic>&#x003C;.05 (<xref ref-type="fig" rid="figure2">Figure 2</xref>). kimi-k2-thinking outperformed deepseek-v3.2 with <italic>P</italic>=.01. All pairwise <italic>P</italic> values and counts for pairwise comparisons are shown in Table S4 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p><table-wrap id="t1" position="float"><label>Table 1.</label><caption><p>Question validity rates by generator after large language model (LLM)-as-judge filtering. Acceptance requires both the majority of judges (at least 4 of 6) to rate the question as logically valid and the majority of judges (at least 4 of 6) to assign a medical accuracy score of 3 or greater.</p></caption><table id="table1" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Generator model</td><td align="left" valign="bottom">Accepted, n/N (%, Wilson 95% CI)</td></tr></thead><tbody><tr><td align="left" valign="top">gpt-5.1-2025-11-13</td><td align="left" valign="top">55/58 (94.8, 85.9%-98.2%)</td></tr><tr><td align="left" valign="top">gemini-3-pro-preview</td><td align="left" valign="top">60/65 (92.3, 83.2%-96.7%)</td></tr><tr><td align="left" valign="top">claude-opus-4-5-20251101</td><td align="left" valign="top">55/66 (83.3, 72.6%-90.4%)</td></tr><tr><td align="left" valign="top">kimi-k2-thinking</td><td align="left" valign="top">67/105 (63.8, 54.3%-72.4%)</td></tr><tr><td align="left" valign="top">grok-4&#x2010;0709</td><td align="left" valign="top">51/91 (56.0, 45.8%-65.8%)</td></tr><tr><td align="left" valign="top">deepseek-v3.2</td><td align="left" valign="top">57/124 (46.0, 37.4%-54.7%)</td></tr></tbody></table></table-wrap><fig position="float" id="figure2"><label>Figure 2.</label><caption><p>Heatmap of Benjamini-Hochberg (BH)&#x2013;adjusted <italic>P</italic> values from pairwise 2-sample proportion tests comparing the validity rates of the 6 language models. Color intensity represents &#x2212;log<sub>10</sub>(BH-adjusted <italic>P</italic>), with darker blue indicating stronger evidence of differences between generator pairs. *Indicates statistically significant differences (BH-adjusted <italic>P</italic>&#x003C;.05). All pairwise <italic>P</italic> values and counts for pairwise comparisons are shown in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig02.png"/></fig><p>Next, we used LOOCV to evaluate each judge by holding it out of the judge ensemble, forming a 5-judge consensus (N=5) from the remaining models, and measuring agreement with that consensus over all questions. For logical validity (<xref ref-type="table" rid="table2">Table 2</xref>), agreement is reported as accuracy, defined as the fraction of items on which the held-out judge matches the ensemble&#x2019;s logical validity binary pass or fail decision. Accuracies range from approximately 0.72 to 0.87, indicating moderate consistency with the ensemble. For the medical accuracy score (<xref ref-type="table" rid="table2">Table 2</xref>), agreement is the fraction of items for which the held-out judge exactly matches the consensus median score. Agreement rates ranged from 0.64 to 0.79, reflecting broadly consistent scoring across judges with mild disagreement around the ensemble median. Overall, we did not observe any outlier judges for both logical validity and medical accuracy evaluation dimensions.</p><table-wrap id="t2" position="float"><label>Table 2.</label><caption><p>Leave-one-out cross-validation (LOOCV) agreement of individual judges with the 5-judge consensus. Logical-validity agreement represents agreement with the ensemble binary pass/fail decision; medical-score agreement represents an exact match to the ensemble median score.</p></caption><table id="table2" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Judge model</td><td align="left" valign="bottom">Logical-validity agreement</td><td align="left" valign="bottom">Medical-score agreement</td></tr></thead><tbody><tr><td align="left" valign="top">gpt-5.1-2025-11-13</td><td align="left" valign="top">0.870</td><td align="left" valign="top">0.769</td></tr><tr><td align="left" valign="top">gemini-3-pro-preview</td><td align="left" valign="top">0.862</td><td align="left" valign="top">0.729</td></tr><tr><td align="left" valign="top">grok-4&#x2010;0709</td><td align="left" valign="top">0.832</td><td align="left" valign="top">0.787</td></tr><tr><td align="left" valign="top">kimi-k2-thinking</td><td align="left" valign="top">0.818</td><td align="left" valign="top">0.747</td></tr><tr><td align="left" valign="top">claude-opus-4-5-20251101</td><td align="left" valign="top">0.788</td><td align="left" valign="top">0.715</td></tr><tr><td align="left" valign="top">deepseek-v3.2</td><td align="left" valign="top">0.725</td><td align="left" valign="top">0.638</td></tr></tbody></table></table-wrap><p>We then performed a public-source similarity audit comparing the 300 selected MedEvalArena questions with 206,578 public comparison items or documents from MedQA, MedMCQA, PubMedQA, MMLU medical subsets, and the official public USMLE step 1 and step 2 clinical knowledge sample-question pages [<xref ref-type="bibr" rid="ref29">29</xref>-<xref ref-type="bibr" rid="ref34">34</xref>]. The audit found no normalized exact-match pairs and no near-duplicate or verbatim candidate pairs.</p><p>Lastly, we assessed possible self-judging bias, whereby a judge model may be biased toward rating its own output more favorably. This judge-by-generator analysis evaluated self-family vs nonself-judge pass rates for all candidate items created by all question generation models. In an adjusted judge&#x2013;item-level interaction fixed-effects model, generator-specific same-family coefficients ranged from approximately 0.0 to +11.0 PPs after accounting for item and judge effects, although none remained statistically significant after BH adjustment (Table S5 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p></sec><sec id="s3-3"><title>Model MCQ Accuracy and Generated MCQ Difficulty</title><p>In the evaluation phase, the accuracy of each examination-taker was assessed on the same 300-question validated set. Answer accuracy ranged from 85.7% (257/300) for kimi-k2-thinking to 91.7% (275/300) for claude-opus-4-5-20251101 (<xref ref-type="table" rid="table3">Table 3</xref>). Model performance did not differ significantly as assessed by a binomial-logit GLMM accounting for examination and item clustering, with a likelihood-ratio test (LRT) <italic>&#x03C7;</italic>&#x00B2;<sub>5</sub>=10.4, <italic>P</italic>=.07. The largest contrast was claude-opus-4-5-20251101 vs kimi-k2-thinking (odds ratio 2.86, 95% CI 0.999&#x2010;8.2; BH-adjusted <italic>P</italic>=.051). Given that both this contrast and the omnibus LRT did not reach the &#x03B1;=.05 threshold, this contrast is reported, at most, as exploratory. The remaining paired model contrasts also did not reach statistical significance (Table S6 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>). Post hoc Monte Carlo simulation using Cochran Q test estimated approximately 80% power to detect a between-model accuracy spread of 9.2% in the 300-question, paired, 6-model design at &#x03B1;=.05, indicating that the study was underpowered for the small differences observed in model performance. Therefore, the observed nonsignificant contrasts should be interpreted as limited evidence to detect differences rather than evidence of equivalence.</p><table-wrap id="t3" position="float"><label>Table 3.</label><caption><p>Accuracy of answer models on the full 300-question validated examination.</p></caption><table id="table3" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Answer model</td><td align="left" valign="bottom">Accuracy, n (%)</td></tr></thead><tbody><tr><td align="left" valign="top">claude-opus-4-5-20251101</td><td align="left" valign="top">275 (91.7)</td></tr><tr><td align="left" valign="top">gpt-5.1-2025-11-13</td><td align="left" valign="top">266 (88.7)</td></tr><tr><td align="left" valign="top">deepseek-v3.2</td><td align="left" valign="top">265 (88.3)</td></tr><tr><td align="left" valign="top">grok-4&#x2010;0709</td><td align="left" valign="top">264 (88)</td></tr><tr><td align="left" valign="top">gemini-3-pro-preview</td><td align="left" valign="top">261 (87)</td></tr><tr><td align="left" valign="top">kimi-k2-thinking</td><td align="left" valign="top">257 (85.7)</td></tr></tbody></table></table-wrap><p>The generator-by-answerer matrix shows modest variation across answer model &#x00D7; generator pairs for frontier LLMs (<xref ref-type="fig" rid="figure3">Figure 3</xref>). There was no evidence of a generator-by-answerer interaction, with a nonsignificant interaction LRT (<italic>&#x03C7;</italic>&#x00B2;<sub>25</sub>=30.1, <italic>P</italic>=.22) and no BH-adjusted cell-level residuals reaching significance. In the question generation phase, claude-opus-4-5-20251101 generated the most challenging examinations (average answer accuracy of 253/300, 84.3%), whereas gemini-3-pro-preview generated the least challenging examinations (289/300, 96.3%). We did not observe a consistent answer-side self-family advantage: diagonal same-family cells were not uniformly higher than off-diagonal cells.</p><fig position="float" id="figure3"><label>Figure 3.</label><caption><p>Generator &#x00D7; answer model accuracy heatmap. Accuracy (%) for each generator-answer model pair on the 300-question examination. The rightmost column and bottom row present the mean accuracy across answer models and generators, respectively. The bottom-right cell presents the overall mean accuracy (88.2%, SD 5.5% PP).</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig03.png"/></fig><p>Next, we compared performance across surgical, medical, and other specialty groups (<xref ref-type="fig" rid="figure4">Figure 4</xref>). Exploratory pairwise comparisons showed differences of &#x2212;2.3 PPs for surgical vs medical, &#x2212;3.8 PPs for surgical vs other, and &#x2212;1.5 PPs for medical vs other specialties. None of these differences was statistically significant (5000 generator-stratified permutations; unadjusted 2-sided <italic>P</italic>=.46, .34, and .64 for surgical vs medical, surgical vs other, and medical vs other specialties, respectively; BH-adjusted <italic>P</italic>=.64 for each comparison). Individual specialty estimates varied widely, but several extreme values were based on limited question attempts. For example, rheumatology/musculoskeletal reached 100% (36/36) accuracy, whereas neurological surgery had relatively low accuracy; however, each contributed only 1 question per generator, corresponding to 6 unique questions and 36 total answer attempts across the 6 answering models. These low-count topic estimates should therefore be interpreted cautiously. To determine whether question difficulty differed by specialty, we conducted a global permutation ANOVA on mean question correctness and found no significant differences across specialties, after accounting for unequal numbers of questions per topic (5000 quiz-stratified permutations; <italic>F</italic><sub>30,269</sub>=.832, &#x03B7;<sup>2</sup>=.085, <italic>P</italic>=.71).</p><fig position="float" id="figure4"><label>Figure 4.</label><caption><p>Forest plot of topic-level accuracy across 31 specialties, grouped into surgical, medical, and other specialties. Points represent pooled percentage accuracy across all 6 examination-takers, with horizontal bars indicating 95% CI. Circle size is proportional to the number of question attempts for each topic, and adjacent columns report correct responses over total attempts and the accuracy percentage. Abbreviations for specialties are shown in Table S7 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig04.png"/></fig></sec><sec id="s3-4"><title>Comparison to Human Physician Performance</title><p>For comparison to human physician performance, we used a 5-physician rater, 6-stratum design with 90 unique questions, stratified by LLM generator (15 sampled per generator/stratum from 50), with 60 questions double-rated (2 raters each) and 30 questions single-rated (1 rater each). Across the 90 sampled questions, we computed overall primary accuracy as the proportion of questions labeled correct at the question level, with single-rated items using the sole rater&#x2019;s label and double-rated items using the consensus label when raters agree and the adjudicated label when they disagree. The overall primary physician accuracy was 85.6% (95% CI 79.4%&#x2010;91.7%; generator-stratified normal approximation with finite-population correction). Accuracy was 73.3% (22/30; Wilson 95% CI 55.6%&#x2010;85.8%) for single-rated items and 91.7% (55/60; Wilson 95% CI 81.9%&#x2010;96.4%) for double-rated items after adjudication. For the paired human-LLM comparison, we compared physician question-level correctness with aggregate LLM correctness on the same 90 questions. The aggregate LLM label was coded correct when at least half of the LLMs answered the question correctly. Aggregate LLM accuracy was 94.4% (85/90; Wilson 95% CI 87.6%&#x2010;97.6%). The paired comparison with physician question-level outcomes was not statistically significant (exact McNemar test, <italic>P</italic>=.08). We note that the observed 8.9 PP LLM-human difference was smaller than the 14 to 15 PP difference that the LLM-human comparison was powered to detect.</p><p>Across generator strata, physician primary accuracy ranged from 73.3% (11/15) to 93.3% (14/15). Humans performed best when the question generator was grok-4&#x2010;0709 (93.3%, 95% CI 82.4%&#x2010;100.0%) and performed worst when the question generator was kimi-k2-thinking (73.3%, 95% CI 54.0%&#x2010;92.7%). For doubly rated shared items (N=60), human responses showed moderate concordance, with a 68.3% (n=41) agreement rate, a 31.7% (n=19) disagreement rate, and a Gwet first-order agreement coefficient (AC1) of 0.50, indicating moderate agreement beyond chance. In addition, all physician raters were asked to flag questions as potentially medically inaccurate or illogical. For questions that were flagged by raters as medically inaccurate or illogical, we adjudicated them through review by an additional physician who had access to OpenEvidence. Of the 90 questions, 17 (18.9%) were initially flagged for review (16 by 1 rater and 1 by 2 raters), and all 17 were found on subsequent adjudication to be consistent with the original generator-specified correct answer.</p></sec><sec id="s3-5"><title>Cost Evaluation and Pareto Frontier Analysis</title><p>Next, we evaluated answering-only inference cost vs accuracy trade-offs, including weaker-reasoning answer models from the same vendor families: grok-4&#x2010;1-fast-reasoning, claude-haiku-4-5-20251001, gemini-2.5-flash-lite, and gpt-5-nano-2025-08-07. The complete answering-only cost-analysis roster is provided in Table S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. While accuracy differed modestly across models (228/300, 76% to 275/300, 91.7%), answering-only per-examination cost varied by nearly 2 orders of magnitude (US $0.19&#x2010;$9.27; <xref ref-type="fig" rid="figure5">Figure 5</xref>). The resulting answering-only cost Pareto frontier consisted of grok-4&#x2010;1-fast-reasoning (259/300, 86.3%; US $0.19), deepseek-v3.2 (265/300, 88.3%; US $0.42), gpt-5.1-2025-11-13 (266/300, 88.7%; US $1.86), and claude-opus-4-5-20251101 (275/300, 91.7%; US $2.15), with all other models being dominated, with higher cost at equal or lower accuracy (<xref ref-type="fig" rid="figure5">Figure 5</xref>). Question generation costs also followed similar cost profiles, with claude-opus-4-5-20251101, gpt-5.1-2025-11-13, and grok-4&#x2010;0709 demonstrating the highest costs (Table S8 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p><fig position="float" id="figure5"><label>Figure 5.</label><caption><p>Accuracy vs cost per examination for large language models (LLMs). Cost per examination represents the cost incurred by each LLM to complete the entire 300-question examination. Accuracy differed only modestly across models; however, per-examination cost varied by nearly 2 orders of magnitude. The Pareto frontier (gray dashed line) consisted of grok-4&#x2010;1-fast-reasoning (accuracy 259/300, 86.3%; cost per examination US $0.19), deepseek-v3.2 (265/300, 88.3%; US $0.42), gpt-5.1-2025-11-13 (266/300, 88.7%; US $1.86), and claude-opus-4-5-20251101 (275/300, 91.7%; US $2.15). Costs were calculated based on model usage from October 1, 2025, to November 30, 2025.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig05.png"/></fig></sec><sec id="s3-6"><title>Medical Accuracy Score Sensitivity Analysis</title><p>In a retrospective sensitivity analysis, we evaluated the effect of varying the minimum medical accuracy score from the judge-ensemble panel on answerer accuracy. All questions met the baseline criterion of a medical accuracy score of 3 or greater. Of the 300 questions, 297 (99%) had judge-ensemble panel majority score of 4 or greater, 232 (77.3%) had judge-ensemble panel majority score of 5, and 79 (26.3%) met strict unanimous criterion, requiring all 6 judges to assign a score of 5 (Table S9 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>). Increased medical accuracy score cutoff was associated with higher answerer accuracy (<xref ref-type="table" rid="table4">Table 4</xref>). In the comparison of judge-ensemble panel majority scores with cutoffs of 4 vs 5 comparison, among the 6 primary generator models, mean accuracy increased from 88.5% to 91.2% (difference 2.7 PP; bootstrap 95% CI 1.3&#x2010;4.7; BH-adjusted <italic>P</italic>=.006). Across all 10 answerer models, mean accuracy increased from 85.5% to 88.9% (difference 3.4 pp; bootstrap 95% CI 1.8&#x2010;5.5; BH-adjusted <italic>P</italic>=.003). The observed differences were larger under the unanimous criterion. Applying the unanimous criterion further increased mean accuracy from 91.2% to 98.3% (difference 7.1 pp; bootstrap 95% CI 4.0&#x2010;9.2; BH-adjusted <italic>P</italic>&#x003C;.001) among the 6 primary generator models, and from 88.9% to 96.0% (difference 7.1 pp; bootstrap 95% CI 3.5&#x2010;9.2; BH-adjusted <italic>P</italic>&#x003C;.001) across all 10 models. Thus, unanimous judge agreement identified a significantly easier subset of questions, with a tendency to be associated with higher medical accuracy score.</p><table-wrap id="t4" position="float"><label>Table 4.</label><caption><p>Sensitivity of answerer-model accuracy to different medical accuracy score thresholds.<sup><xref ref-type="table-fn" rid="table4fn1">a</xref></sup></p></caption><table id="table4" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Answerer model</td><td align="left" valign="bottom">Accuracy at score 4 or greater (N=297; 95% CI)</td><td align="left" valign="bottom">Accuracy at score 5 (N=232; 95% CI)</td><td align="left" valign="bottom">Cutoff 4 vs 5 accuracy PP<sup><xref ref-type="table-fn" rid="table4fn2">b</xref></sup> difference<sup><xref ref-type="table-fn" rid="table4fn3">c</xref></sup> (95% CI)</td><td align="left" valign="bottom">Accuracy at unanimous score (N=79; 95% CI)</td><td align="left" valign="bottom">Unanimous (6/6) vs score 5 accuracy PP difference<sup><xref ref-type="table-fn" rid="table4fn3">c</xref></sup> (95% CI)</td></tr></thead><tbody><tr><td align="left" valign="top">Primary 6-model mean</td><td align="left" valign="top">88.5 (86.0-91.1)</td><td align="left" valign="top">91.2 (88.7-94.3)</td><td align="left" valign="top">2.7 (+1.3 to +4.7)</td><td align="left" valign="top">98.3 (95.0-99.8)</td><td align="left" valign="top">7.1 (+4.0 to +9.2)</td></tr><tr><td align="left" valign="top">claude-opus-4-5-20251101</td><td align="left" valign="top">92.3 (89.2-95.1)</td><td align="left" valign="top">94.6 (91.6-97.8)</td><td align="left" valign="top">2.3 (+0.3 to +5.0)</td><td align="left" valign="top">100 (100-100)</td><td align="left" valign="top">5.4 (+2.3 to +8.5)</td></tr><tr><td align="left" valign="top">gpt-5.1-2025-11-13</td><td align="left" valign="top">88.9 (85.6-92.7)</td><td align="left" valign="top">91.9 (88.7-95.7)</td><td align="left" valign="top">3.0 (+0.8 to +5.6)</td><td align="left" valign="top">98.1 (95.5-100.0)</td><td align="left" valign="top">6.2 (+2.2 to +9.9)</td></tr><tr><td align="left" valign="top">deepseek-v3.2</td><td align="left" valign="top">88.5 (85.2-91.9)</td><td align="left" valign="top">90.7 (86.9-94.6)</td><td align="left" valign="top">2.2 (+0.3 to +4.4)</td><td align="left" valign="top">97.1 (91.7-100.0)</td><td align="left" valign="top">6.4 (+2.0 to +9.8)</td></tr><tr><td align="left" valign="top">grok-4&#x2010;0709</td><td align="left" valign="top">88.2 (83.9-92.3)</td><td align="left" valign="top">90.2 (85.9-94.8)</td><td align="left" valign="top">2.0 (+0.2 to +4.8)</td><td align="left" valign="top">96.6 (89.3-100.0)</td><td align="left" valign="top">6.5 (+0.4 to +9.9)</td></tr><tr><td align="left" valign="top">gemini-3-pro-preview</td><td align="left" valign="top">87.2 (83.6-91.1)</td><td align="left" valign="top">90.9 (87.4-95.3)</td><td align="left" valign="top">3.8 (+1.8 to +6.5)</td><td align="left" valign="top">99.3 (97.2-100.0)</td><td align="left" valign="top">8.3 (+4.2 to +11.6)</td></tr><tr><td align="left" valign="top">kimi-k2-thinking</td><td align="left" valign="top">85.8 (82.0-89.4)</td><td align="left" valign="top">88.8 (84.8-92.8)</td><td align="left" valign="top">2.9 (+0.7 to +6.0)</td><td align="left" valign="top">98.6 (94.4-100.0)</td><td align="left" valign="top">9.8 (+5.8 to +13.3)</td></tr><tr><td align="left" valign="top">grok-4&#x2010;1-fast-reasoning</td><td align="left" valign="top">86.5 (82.9-89.9)</td><td align="left" valign="top">90.3 (86.1-94.2)</td><td align="left" valign="top">3.8 (+1.5 to +6.3)</td><td align="left" valign="top">99.3 (97.2-100.0)</td><td align="left" valign="top">9.0 (+5.7 to +12.4)</td></tr><tr><td align="left" valign="top">gpt-5-nano-2025-08-07</td><td align="left" valign="top">80.8 (76.0-85.6)</td><td align="left" valign="top">86.3 (82.5-90.7)</td><td align="left" valign="top">5.5 (+2.7 to +8.9)</td><td align="left" valign="top">93.1 (85.2-98.4)</td><td align="left" valign="top">6.8 (+0.1 to +11.0)</td></tr><tr><td align="left" valign="top">claude-haiku-4-5-20251001</td><td align="left" valign="top">80.4 (75.0-85.7)</td><td align="left" valign="top">85.3 (79.4-91.1)</td><td align="left" valign="top">4.9 (+2.1 to +8.0)</td><td align="left" valign="top">89.4 (76.9-96.1)</td><td align="left" valign="top">4.1 (&#x2013;9.3 to +10.2)</td></tr><tr><td align="left" valign="top">gemini-2.5-flash-lite</td><td align="left" valign="top">76.4 (71.5-81.1)</td><td align="left" valign="top">80.3 (75.2-85.6)</td><td align="left" valign="top">3.9 (+0.6 to +7.9)</td><td align="left" valign="top">88.4 (78.0-96.6)</td><td align="left" valign="top">8.1 (&#x2013;1.5 to +16.5)</td></tr><tr><td align="left" valign="top">All 10-model mean</td><td align="left" valign="top">85.5 (82.8-88.3)</td><td align="left" valign="top">88.9 (86.0-92.1)</td><td align="left" valign="top">3.4 (+1.8 to +5.5)</td><td align="left" valign="top">96.0 (92.5-98.0)</td><td align="left" valign="top">7.1 (+3.5 to +9.2)</td></tr></tbody></table><table-wrap-foot><fn id="table4fn1"><p><sup>a</sup>Accuracy was calculated for questions retained at thresholds of &#x2265;4, &#x2265;5, and a unanimous score of 5. The reported values represent unweighted means of the 6 generator-specific accuracies. The column N values indicate the total number of retained questions across generators. Accuracy differences are reported in PPs. The 95% CIs for these differences were estimated using 10,000 generator-stratified bootstrap replicates.</p></fn><fn id="table4fn2"><p><sup>b</sup>PP: percentage point.</p></fn><fn id="table4fn3"><p><sup>c</sup>PP differences were calculated from unrounded accuracy values and then rounded to 1 decimal place. They may differ by 0.1 PPs from differences between the displayed accuracies.</p></fn></table-wrap-foot></table-wrap><p>Next, we explored whether this effect could be explained in part by an association between model tendency to assign a medical accuracy score of 5 and answer correctness. For each of the 6 generator models, we analyzed concordance between the medical-accuracy rating and the correctness of that model&#x2019;s answer to the same item using a linear probability model. When answering a question correctly, 4 models were significantly more likely to have given that question a medical accuracy score of 5: claude-opus-4-5-20251101 by 32.6 PPs (95% CI 13.6&#x2010;53.5), gemini-3-pro-preview by 29 PPs (95% CI 7.3&#x2010;49.1), deepseek-v3.2 by 21.1 PPs (95% CI 5.5&#x2010;38.8), and gpt-5.1-2025-11-13 by 17.1 PPs (95% CI 6.0&#x2010;30.4). These findings remained significant after BH correction (adjusted <italic>P</italic>=.01-.02). There was no clear association for grok-4&#x2010;0709 (1.2 PPs; 95% CI &#x2212;11.9 to 13.8) or kimi-k2-thinking (12.4 PPs; 95% CI &#x2212;4.0 to 27.0). Judge-answer model effect sizes are visualized in <xref ref-type="fig" rid="figure6">Figure 6</xref>. Overall, the higher model accuracy observed for questions unanimously rated as 5 may partly reflect a model-specific association between answering a question correctly and being more likely to assign it the highest medical accuracy score. However, given the post hoc nature of this analysis, these retrospective findings do not demonstrate causation.</p><fig position="float" id="figure6"><label>Figure 6.</label><caption><p>Adjusted judge-answer concordance by model. Points indicate the adjusted difference, in percentage points, in the probability of assigning a medical accuracy score of 5 to questions answered correctly vs incorrectly. Error bars indicate 95% bootstrap CIs.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="jmir_v28i1e96885_fig06.png"/></fig></sec></sec><sec id="s4" sec-type="discussion"><title>Discussion</title><sec id="s4-1"><title>Principal Results</title><p>We used symmetric round-robin peer-assessment to assess LLM medical reasoning across both question generation and answering. Frontier models achieved similarly high question-answering (QA) accuracy with no statistically significant omnibus difference. Although not powered as a formal equivalence or noninferiority study, these findings suggest that medical board&#x2013;style MCQ answering alone may provide limited discrimination at current performance levels of frontier models. Notably, we found differences in generator validity rates, suggesting that reasoning processes supporting question generation capability may be a more discriminative task for assessing medical reasoning performance than answering accuracy alone. Finally, we found that while the spread in model accuracy was modest, per-examination inference cost differed by nearly 2 orders of magnitude across models, producing a Pareto frontier among frontier models.</p><p>The finding that question generation quality separates models more than examination-taking accuracy supports the view that QA and question-generation involve distinct cognitive and reasoning capabilities [<xref ref-type="bibr" rid="ref38">38</xref>]. Generation requires not only medical knowledge but also the ability to pose well-specified clinical vignettes with logical coherence [<xref ref-type="bibr" rid="ref39">39</xref>], necessitating a uniquely defensible single best answer and avoiding underspecification [<xref ref-type="bibr" rid="ref40">40</xref>]. The reasoning processes supporting these facets of question generation are comparatively more recursive and nonlinear than QA [<xref ref-type="bibr" rid="ref41">41</xref>], and may be more difficult for LLMs due to their relative deficiencies in planning [<xref ref-type="bibr" rid="ref42">42</xref>]. Question-generation capability reflects underlying medical reasoning and specification ability [<xref ref-type="bibr" rid="ref43">43</xref>] and thus is relevant to clinical reliability, as underspecification or internal inconsistency may lead to ambiguous or misleading medical guidance. Nonetheless, we acknowledge that examination-taking performance and question-generation ability are only proxy measures of real-world clinical reasoning.</p></sec><sec id="s4-2"><title>Comparison With Prior Work</title><p>Our approach extends prior dynamic benchmarking efforts, including DynaBench and LiveBench, which address benchmark saturation by supporting regenerable, updated evaluations [<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref13">13</xref>]. In addition, it builds on automated benchmark construction, such as that used by BenchBuilder, which uses LLM-based pipelines to filter and validate benchmark prompts at scale [<xref ref-type="bibr" rid="ref14">14</xref>]. MedEvalArena is most similar to the reciprocal, round-robin evaluation approach introduced in Autobench [<xref ref-type="bibr" rid="ref15">15</xref>], while being focused on medical reasoning. Our approach also differs from Autobench in that the multiple-choice nature of MedEvalArena uses the LLM-as-judge phase immediately after question-generation to validate question coherence rather than response scoring. This structure provides a scalable safeguard against incoherent or clinically inaccurate questions while promoting the regenerability of dynamic benchmarking.</p></sec><sec id="s4-3"><title>Limitations</title><p>We acknowledge several limitations of this work. First, the study was underpowered to detect small differences in LLM examination-taking performance. Specifically, based on post hoc simulations of the Cochran Q test, the study had less than 80% omnibus power to detect observed differences in model accuracy of less than approximately 9.2 PPs. Power calculations suggest that detecting a 3.0 PP difference centered on the observed mean accuracy of 88.2% (1588/1800) would require approximately 2900 questions to achieve 80% power, or about 3000 questions in a balanced 6-generator design with 500 questions per generator. Accordingly, future work will require substantially larger item banks to detect frontier model differences.</p><p>Second, we acknowledge that the physician comparison was limited in scope and should be interpreted as a descriptive human reference for contextualizing question difficulty, not as a definitive noninferiority or equivalence study. Detecting a smaller 3.0 PP physician vs LLM difference would require more than 2600 paired question-level comparisons, corresponding to 4500 physician responses, or 900 responses per physician, assuming a 5-physician panel and using the presented stratified-block methodology. Thus, future work will warrant additional physician involvement to provide a robust comparison of physician vs LLM performance.</p><p>Third, the LLM as judge paradigm may introduce potential biases, such as shared training artifacts between LLM families or self-judging bias. To mitigate these biases, future work may include the use of a larger judge ensemble or the exclusion of self-judging. Human rating of question validity as performed in this study may be helpful; however, due to the scalable and dynamically regenerable design of MedEvalArena, human oversight may necessarily remain partial or limited. Here, physician comparison was constrained in that, by design, it did not include systematic physician assessment of rejected LLM-as-judge questions. Additionally, there was only modest agreement among physician answers on the generated question dataset. The latter 2 points limit the extent to which the physician comparison can be treated as a definitive gold standard.</p><p>Lastly, the present work evaluates a single snapshot generated with a fixed model roster and is thus susceptible to knowledge drift. To mitigate against knowledge drift, MedEvalArena is designed to support dynamic regeneration of updated evaluations. The full pipeline is provided to facilitate future instantiations of MedEvalArena.</p></sec><sec id="s4-4"><title>Conclusions</title><p>MedEvalArena establishes a scalable, regenerable, peer-judged framework for benchmarking LLM medical reasoning. Valid question generation emerged as a more differentiating benchmark than QA, revealing limitations in LLM medical reasoning that conventional QA tasks may fail to capture. Future larger-scale instantiations with expanded physician oversight are needed to improve the statistical power for tracking model performance as LLM capabilities evolve.</p></sec></sec></body><back><notes><sec><title>Generative AI Use</title><p>The authors declare the use of generative AI (GenAI) in the research and writing process. According to the Generative AI Delegation Taxonomy (GAIDeT; 2025), the following tasks were delegated to GenAI tools under full human supervision: code generation, data visualization, proofreading, and editing. The GenAI tool used was ChatGPT-5.5. The authors take full responsibility for the final manuscript. GenAI tools are not listed as authors and do not bear responsibility for the final outcomes. This declaration was submitted by all the authors.</p></sec><sec><title>Funding</title><p>The authors declared no financial support was received for this work.</p></sec><sec><title>Data Availability</title><p>The MedEvalArena dataset and analysis code are available in the MedEvalArena GitHub repository [<xref ref-type="bibr" rid="ref44">44</xref>], with the latest GitHub snapshot available on Zenodo [<xref ref-type="bibr" rid="ref45">45</xref>].</p></sec></notes><fn-group><fn fn-type="con"><p>Conceptualization: FL (supporting), AA (supporting), DB (lead)</p><p>Data curation: PP (supporting), DB (supporting)</p><p>Formal analysis: DB (lead)</p><p>Methodology: AA (supporting), DB (lead)</p><p>Writing &#x2013; original draft: PP (lead), KS (supporting), GC (supporting), VK (supporting), EW (supporting), DB (supporting)</p><p>Writing &#x2013; review &#x0026; editing: FL (supporting), AA (supporting), DB (supporting)</p></fn><fn fn-type="conflict"><p>None declared.</p></fn></fn-group><glossary><title>Abbreviations</title><def-list><def-item><term id="abb1">AAMC</term><def><p>Association of American Medical Colleges</p></def></def-item><def-item><term id="abb2">ABFM</term><def><p>American Board of Family Medicine</p></def></def-item><def-item><term id="abb3">ABIM</term><def><p>American Board of Internal Medicine</p></def></def-item><def-item><term id="abb4">ABMS</term><def><p>American Board of Medical Specialties</p></def></def-item><def-item><term id="abb5">BH</term><def><p>Benjamini-Hochberg</p></def></def-item><def-item><term id="abb6">GLMM</term><def><p>generalized linear mixed model</p></def></def-item><def-item><term id="abb7">IRB</term><def><p>Institutional Review Board</p></def></def-item><def-item><term id="abb8">LLM</term><def><p>large language model</p></def></def-item><def-item><term id="abb9">LOOCV</term><def><p>leave-one-out cross-validation</p></def></def-item><def-item><term id="abb10">LRT</term><def><p>likelihood-ratio test</p></def></def-item><def-item><term id="abb11">mARC-QA</term><def><p>Medical Abstraction and Reasoning Corpus Question and Answer</p></def></def-item><def-item><term id="abb12">MCQ</term><def><p>multiple-choice question</p></def></def-item><def-item><term id="abb13">MedMCQA</term><def><p>medical multiple-choice question-answering</p></def></def-item><def-item><term id="abb14">MedQA</term><def><p>medical question-answering</p></def></def-item><def-item><term id="abb15">MMLU</term><def><p>Massive Multitask Language Understanding</p></def></def-item><def-item><term id="abb16">PP</term><def><p>percentage point</p></def></def-item><def-item><term id="abb17">PubMedQA</term><def><p>PubMed question-answering</p></def></def-item><def-item><term id="abb18">QA</term><def><p>question answering</p></def></def-item><def-item><term id="abb19">UCSF</term><def><p>University of California, San Francisco</p></def></def-item><def-item><term id="abb20">UKMLA</term><def><p>United Kingdom Medical Licensing Assessment</p></def></def-item><def-item><term id="abb21">USMLE</term><def><p>United States Medical Licensing Examination</p></def></def-item></def-list></glossary><ref-list><title>References</title><ref id="ref1"><label>1</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Brin</surname><given-names>D</given-names> </name><name name-style="western"><surname>Sorin</surname><given-names>V</given-names> </name><name name-style="western"><surname>Vaid</surname><given-names>A</given-names> </name><etal/></person-group><article-title>Comparing ChatGPT and GPT-4 performance in USMLE soft skill assessments</article-title><source>Sci Rep</source><year>2023</year><month>10</month><day>1</day><volume>13</volume><issue>1</issue><fpage>16492</fpage><pub-id pub-id-type="doi">10.1038/s41598-023-43436-9</pub-id><pub-id pub-id-type="medline">37779171</pub-id></nlm-citation></ref><ref id="ref2"><label>2</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Jiang</surname><given-names>LY</given-names> </name><name name-style="western"><surname>Liu</surname><given-names>XC</given-names> </name><name name-style="western"><surname>Nejatian</surname><given-names>NP</given-names> </name><etal/></person-group><article-title>Health system-scale language models are all-purpose prediction engines</article-title><source>Nature</source><year>2023</year><month>07</month><volume>619</volume><issue>7969</issue><fpage>357</fpage><lpage>362</lpage><pub-id pub-id-type="doi">10.1038/s41586-023-06160-y</pub-id><pub-id pub-id-type="medline">37286606</pub-id></nlm-citation></ref><ref id="ref3"><label>3</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Peng</surname><given-names>C</given-names> </name><name name-style="western"><surname>Yang</surname><given-names>X</given-names> </name><name name-style="western"><surname>Chen</surname><given-names>A</given-names> </name><etal/></person-group><article-title>A study of generative large language model for medical research and healthcare</article-title><source>NPJ Digit Med</source><year>2023</year><month>11</month><day>16</day><volume>6</volume><issue>1</issue><fpage>210</fpage><pub-id pub-id-type="doi">10.1038/s41746-023-00958-w</pub-id><pub-id pub-id-type="medline">37973919</pub-id></nlm-citation></ref><ref id="ref4"><label>4</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Singhal</surname><given-names>K</given-names> </name><name name-style="western"><surname>Tu</surname><given-names>T</given-names> </name><name name-style="western"><surname>Gottweis</surname><given-names>J</given-names> </name><etal/></person-group><article-title>Toward expert-level medical question answering with large language models</article-title><source>Nat Med</source><year>2025</year><month>03</month><volume>31</volume><issue>3</issue><fpage>943</fpage><lpage>950</lpage><pub-id pub-id-type="doi">10.1038/s41591-024-03423-7</pub-id><pub-id pub-id-type="medline">39779926</pub-id></nlm-citation></ref><ref id="ref5"><label>5</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Bicknell</surname><given-names>BT</given-names> </name><name name-style="western"><surname>Butler</surname><given-names>D</given-names> </name><name name-style="western"><surname>Whalen</surname><given-names>S</given-names> </name><etal/></person-group><article-title>ChatGPT-4 omni performance in USMLE disciplines and clinical skills: comparative analysis</article-title><source>JMIR Med Educ</source><year>2024</year><month>11</month><day>6</day><volume>10</volume><fpage>e63430</fpage><pub-id pub-id-type="doi">10.2196/63430</pub-id><pub-id pub-id-type="medline">39504445</pub-id></nlm-citation></ref><ref id="ref6"><label>6</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Hager</surname><given-names>P</given-names> </name><name name-style="western"><surname>Jungmann</surname><given-names>F</given-names> </name><name name-style="western"><surname>Holland</surname><given-names>R</given-names> </name><etal/></person-group><article-title>Evaluation and mitigation of the limitations of large language models in clinical decision-making</article-title><source>Nat Med</source><year>2024</year><month>09</month><volume>30</volume><issue>9</issue><fpage>2613</fpage><lpage>2622</lpage><pub-id pub-id-type="doi">10.1038/s41591-024-03097-1</pub-id><pub-id pub-id-type="medline">38965432</pub-id></nlm-citation></ref><ref id="ref7"><label>7</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Williams</surname><given-names>CYK</given-names> </name><name name-style="western"><surname>Miao</surname><given-names>BY</given-names> </name><name name-style="western"><surname>Kornblith</surname><given-names>AE</given-names> </name><name name-style="western"><surname>Butte</surname><given-names>AJ</given-names> </name></person-group><article-title>Evaluating the use of large language models to provide clinical recommendations in the emergency department</article-title><source>Nat Commun</source><year>2024</year><month>10</month><day>8</day><volume>15</volume><issue>1</issue><fpage>8236</fpage><pub-id pub-id-type="doi">10.1038/s41467-024-52415-1</pub-id><pub-id pub-id-type="medline">39379357</pub-id></nlm-citation></ref><ref id="ref8"><label>8</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Kim</surname><given-names>J</given-names> </name><name name-style="western"><surname>Podlasek</surname><given-names>A</given-names> </name><name name-style="western"><surname>Shidara</surname><given-names>K</given-names> </name><name name-style="western"><surname>Liu</surname><given-names>F</given-names> </name><name name-style="western"><surname>Alaa</surname><given-names>A</given-names> </name><name name-style="western"><surname>Bernardo</surname><given-names>D</given-names> </name></person-group><article-title>Limitations of large language models in clinical problem-solving arising from inflexible reasoning</article-title><source>Sci Rep</source><year>2025</year><volume>15</volume><issue>1</issue><fpage>39426</fpage><pub-id pub-id-type="doi">10.1038/s41598-025-22940-0</pub-id></nlm-citation></ref><ref id="ref9"><label>9</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Qiu</surname><given-names>P</given-names> </name><name name-style="western"><surname>Wu</surname><given-names>C</given-names> </name><name name-style="western"><surname>Liu</surname><given-names>S</given-names> </name><etal/></person-group><article-title>Quantifying the reasoning abilities of LLMs on clinical cases</article-title><source>Nat Commun</source><year>2025</year><month>11</month><day>6</day><volume>16</volume><issue>1</issue><fpage>9799</fpage><pub-id pub-id-type="doi">10.1038/s41467-025-64769-1</pub-id><pub-id pub-id-type="medline">41198657</pub-id></nlm-citation></ref><ref id="ref10"><label>10</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Tam</surname><given-names>TYC</given-names> </name><name name-style="western"><surname>Sivarajkumar</surname><given-names>S</given-names> </name><name name-style="western"><surname>Kapoor</surname><given-names>S</given-names> </name><etal/></person-group><article-title>A framework for human evaluation of large language models in healthcare derived from literature review</article-title><source>NPJ Digit Med</source><year>2024</year><month>09</month><day>28</day><volume>7</volume><issue>1</issue><fpage>258</fpage><pub-id pub-id-type="doi">10.1038/s41746-024-01258-7</pub-id><pub-id pub-id-type="medline">39333376</pub-id></nlm-citation></ref><ref id="ref11"><label>11</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Chollet</surname><given-names>F</given-names> </name><name name-style="western"><surname>Knoop</surname><given-names>M</given-names> </name><name name-style="western"><surname>Kamradt</surname><given-names>G</given-names> </name><name name-style="western"><surname>Landers</surname><given-names>B</given-names> </name></person-group><article-title>ARC prize 2024: technical report</article-title><source>arXiv</source><comment>Preprint posted online on  Dec 5, 2024</comment><pub-id pub-id-type="doi">10.48550/arXiv.2412.04604</pub-id></nlm-citation></ref><ref id="ref12"><label>12</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Kiela</surname><given-names>D</given-names> </name><name name-style="western"><surname>Bartolo</surname><given-names>M</given-names> </name><name name-style="western"><surname>Nie</surname><given-names>Y</given-names> </name><etal/></person-group><article-title>Dynabench: rethinking benchmarking in NLP</article-title><source>Proc 2021 Conf North Am Chapter Assoc Comput Linguist</source><year>2021</year><fpage>4110</fpage><lpage>4124</lpage><pub-id pub-id-type="doi">10.18653/v1/2021.naacl-main.324</pub-id></nlm-citation></ref><ref id="ref13"><label>13</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>White</surname><given-names>C</given-names> </name><name name-style="western"><surname>Dooley</surname><given-names>S</given-names> </name><name name-style="western"><surname>Roberts</surname><given-names>M</given-names> </name><etal/></person-group><article-title>LiveBench: a challenging, contamination-free LLM benchmark</article-title><source>arXiv</source><comment>Preprint posted online on  Jun 27, 2024</comment><pub-id pub-id-type="doi">10.48550/arXiv.2406.19314</pub-id></nlm-citation></ref><ref id="ref14"><label>14</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Li</surname><given-names>T</given-names> </name><name name-style="western"><surname>Chiang</surname><given-names>WL</given-names> </name><name name-style="western"><surname>Frick</surname><given-names>E</given-names> </name><etal/></person-group><article-title>From crowdsourced data to high-quality benchmarks: arena-hard and benchbuilder pipeline</article-title><source>arXiv</source><comment>Preprint posted online on  Jun 17, 2024</comment><pub-id pub-id-type="doi">10.48550/arXiv.2406.11939</pub-id></nlm-citation></ref><ref id="ref15"><label>15</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Loi</surname><given-names>D</given-names> </name><name name-style="western"><surname>Mui&#x00E0;</surname><given-names>EM</given-names> </name><name name-style="western"><surname>Siciliano</surname><given-names>F</given-names> </name><etal/></person-group><article-title>AutoBench: automating LLM evaluation through reciprocal peer assessment</article-title><source>arXiv</source><comment>Preprint posted online on  Oct 26, 2025</comment><pub-id pub-id-type="doi">10.48550/arXiv.2510.22593</pub-id></nlm-citation></ref><ref id="ref16"><label>16</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Verga</surname><given-names>P</given-names> </name><name name-style="western"><surname>Hofstatter</surname><given-names>S</given-names> </name><name name-style="western"><surname>Althammer</surname><given-names>S</given-names> </name><etal/></person-group><article-title>Replacing judges with juries: evaluating LLM generations with a panel of diverse models</article-title><source>arXiv</source><comment>Preprint posted online on  Apr 29, 2024</comment><pub-id pub-id-type="doi">10.48550/arXiv.2404.18796</pub-id></nlm-citation></ref><ref id="ref17"><label>17</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Li</surname><given-names>Y</given-names> </name><name name-style="western"><surname>Mohamud</surname><given-names>JH</given-names> </name><name name-style="western"><surname>Sun</surname><given-names>C</given-names> </name><name name-style="western"><surname>Wu</surname><given-names>D</given-names> </name><name name-style="western"><surname>Boulet</surname><given-names>B</given-names> </name></person-group><article-title>Leveraging LLMs as meta-judges: a multi-agent framework for evaluating LLM judgments</article-title><source>arXiv</source><comment>Preprint posted online on  Apr 23, 2025</comment><pub-id pub-id-type="doi">10.48550/arXiv.2504.17087</pub-id></nlm-citation></ref><ref id="ref18"><label>18</label><nlm-citation citation-type="report"><article-title>Artificial analysis state of AI: Q3 2025 highlights report</article-title><year>2025</year><access-date>2026-10-29</access-date><publisher-name>Artificial Analysis</publisher-name><comment><ext-link ext-link-type="uri" xlink:href="https://artificialanalysiscdn.com/public-reports/state-of-ai-2025-q3-highlights-artificial-analysis.pdf?utm_source">https://artificialanalysiscdn.com/public-reports/state-of-ai-2025-q3-highlights-artificial-analysis.pdf?utm_source</ext-link></comment></nlm-citation></ref><ref id="ref19"><label>19</label><nlm-citation citation-type="web"><source>OpenAI</source><access-date>2025-11-30</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://openai.com">https://openai.com</ext-link></comment></nlm-citation></ref><ref id="ref20"><label>20</label><nlm-citation citation-type="web"><source>Anthropic</source><access-date>2025-11-30</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.anthropic.com">https://www.anthropic.com</ext-link></comment></nlm-citation></ref><ref id="ref21"><label>21</label><nlm-citation citation-type="web"><article-title>Gemini 3 pro preview</article-title><source>Google AI Studio</source><access-date>2026-09-14</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://ai.google.dev/gemini-api/docs/models/gemini-3-pro-preview">https://ai.google.dev/gemini-api/docs/models/gemini-3-pro-preview</ext-link></comment></nlm-citation></ref><ref id="ref22"><label>22</label><nlm-citation citation-type="other"><person-group person-group-type="author"><collab>DeepSeek-AI</collab><name name-style="western"><surname>Liu</surname><given-names>A</given-names> </name><name name-style="western"><surname>Mei</surname><given-names>A</given-names> </name><etal/></person-group><article-title>DeepSeek-V3.2: pushing the frontier of open large language models</article-title><source>arXiv</source><comment>Preprint posted online on  Dec 2, 2025</comment><pub-id pub-id-type="doi">10.48550/arXiv.2512.02556</pub-id></nlm-citation></ref><ref id="ref23"><label>23</label><nlm-citation citation-type="web"><source>SpaceX</source><access-date>2025-11-30</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://x.ai">https://x.ai</ext-link></comment></nlm-citation></ref><ref id="ref24"><label>24</label><nlm-citation citation-type="other"><person-group person-group-type="author"><collab>Kimi Team</collab><name name-style="western"><surname>Bai</surname><given-names>Y</given-names> </name><name name-style="western"><surname>Bao</surname><given-names>Y</given-names> </name><etal/></person-group><article-title>Kimi k2: open agentic intelligence</article-title><source>arXiv</source><comment>Preprint posted online on  Jul 28, 2025</comment><pub-id pub-id-type="doi">10.48550/arXiv.2507.20534</pub-id></nlm-citation></ref><ref id="ref25"><label>25</label><nlm-citation citation-type="web"><source>Association of American Medical Colleges (AAMC)</source><year>2025</year><access-date>2025-10-01</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.aamc.org/">https://www.aamc.org/</ext-link></comment></nlm-citation></ref><ref id="ref26"><label>26</label><nlm-citation citation-type="web"><source>American Board of Medical Specialties</source><access-date>2025-10-01</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.abms.org/">https://www.abms.org/</ext-link></comment></nlm-citation></ref><ref id="ref27"><label>27</label><nlm-citation citation-type="web"><article-title>One-day exam content outline</article-title><source>American Board of Family Medicine</source><year>2026</year><access-date>2026-01-05</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.theabfm.org/one-day-exam-content-outline/">https://www.theabfm.org/one-day-exam-content-outline/</ext-link></comment></nlm-citation></ref><ref id="ref28"><label>28</label><nlm-citation citation-type="report"><article-title>Internal medicine certification examination blueprint</article-title><year>2025</year><access-date>2026-01-05</access-date><publisher-name>American Board of Internal Medicine</publisher-name><comment><ext-link ext-link-type="uri" xlink:href="https://www.abim.org/media/wvwoy3wi/internal-medicine-cert.pdf">https://www.abim.org/media/wvwoy3wi/internal-medicine-cert.pdf</ext-link></comment></nlm-citation></ref><ref id="ref29"><label>29</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Jin</surname><given-names>D</given-names> </name><name name-style="western"><surname>Pan</surname><given-names>E</given-names> </name><name name-style="western"><surname>Oufattole</surname><given-names>N</given-names> </name><name name-style="western"><surname>Weng</surname><given-names>WH</given-names> </name><name name-style="western"><surname>Fang</surname><given-names>H</given-names> </name><name name-style="western"><surname>Szolovits</surname><given-names>P</given-names> </name></person-group><article-title>What disease does this patient have? A large-scale open domain question answering dataset from medical exams</article-title><source>Appl Sci</source><year>2021</year><volume>11</volume><fpage>6421</fpage><pub-id pub-id-type="doi">10.3390/app11146421</pub-id></nlm-citation></ref><ref id="ref30"><label>30</label><nlm-citation citation-type="book"><person-group person-group-type="author"><name name-style="western"><surname>Pal</surname><given-names>A</given-names> </name><name name-style="western"><surname>Umapathi</surname><given-names>LK</given-names> </name><name name-style="western"><surname>Sankarasubbu</surname><given-names>M</given-names> </name></person-group><person-group person-group-type="editor"><name name-style="western"><surname>Flores</surname><given-names>G</given-names> </name><name name-style="western"><surname>Chen</surname><given-names>GH</given-names> </name><name name-style="western"><surname>Pollard</surname><given-names>T</given-names> </name><name name-style="western"><surname>Ho</surname><given-names>JC</given-names> </name><name name-style="western"><surname>Naumann</surname><given-names>T</given-names> </name></person-group><article-title>MedMCQA: a large-scale multi-subject multi-choice dataset for medical domain question answering</article-title><source>Proceedings of the Conference on Health, Inference, and Learning</source><year>2022</year><access-date>2026-10-29</access-date><fpage>248</fpage><lpage>260</lpage></nlm-citation></ref><ref id="ref31"><label>31</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Jin</surname><given-names>Q</given-names> </name><name name-style="western"><surname>Dhingra</surname><given-names>B</given-names> </name><name name-style="western"><surname>Liu</surname><given-names>Z</given-names> </name><name name-style="western"><surname>Cohen</surname><given-names>WW</given-names> </name><name name-style="western"><surname>Lu</surname><given-names>X</given-names> </name></person-group><article-title>PubMedQA: a dataset for biomedical research question answering</article-title><source>Proc 2019 Conf Empir Methods Nat Lang Process 9th Int Jt Conf Nat Lang Process</source><year>2019</year><fpage>2567</fpage><lpage>2577</lpage><pub-id pub-id-type="doi">10.18653/v1/D19-1259</pub-id></nlm-citation></ref><ref id="ref32"><label>32</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Hendrycks</surname><given-names>D</given-names> </name><name name-style="western"><surname>Burns</surname><given-names>C</given-names> </name><name name-style="western"><surname>Basart</surname><given-names>S</given-names> </name><etal/></person-group><article-title>Measuring massive multitask language understanding</article-title><source>arXiv</source><comment>Preprint posted online on  Sep 7, 2020</comment><pub-id pub-id-type="doi">10.48550/arXiv.2009.03300</pub-id></nlm-citation></ref><ref id="ref33"><label>33</label><nlm-citation citation-type="web"><article-title>Step 1 sample test questions</article-title><source>United States Medical Licensing Examination (USMLE)</source><access-date>2026-06-06</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.usmle.org/exam-resources/step-1-materials/step-1-sample-test-questions">https://www.usmle.org/exam-resources/step-1-materials/step-1-sample-test-questions</ext-link></comment></nlm-citation></ref><ref id="ref34"><label>34</label><nlm-citation citation-type="web"><article-title>Step 2 CK sample test questions</article-title><source>United States Medical Licensing Examination (USMLE)</source><access-date>2026-06-06</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://www.usmle.org/exam-resources/step-2-ck-materials/step-2-ck-sample-test-questions">https://www.usmle.org/exam-resources/step-2-ck-materials/step-2-ck-sample-test-questions</ext-link></comment></nlm-citation></ref><ref id="ref35"><label>35</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Baayen</surname><given-names>RH</given-names> </name><name name-style="western"><surname>Davidson</surname><given-names>DJ</given-names> </name><name name-style="western"><surname>Bates</surname><given-names>DM</given-names> </name></person-group><article-title>Mixed-effects modeling with crossed random effects for subjects and items</article-title><source>J Mem Lang</source><year>2008</year><month>11</month><volume>59</volume><issue>4</issue><fpage>390</fpage><lpage>412</lpage><pub-id pub-id-type="doi">10.1016/j.jml.2007.12.005</pub-id></nlm-citation></ref><ref id="ref36"><label>36</label><nlm-citation citation-type="book"><person-group person-group-type="author"><name name-style="western"><surname>Wilson</surname><given-names>M</given-names> </name></person-group><source>Explanatory Item Response Models: A Generalized Linear and Nonlinear Approach</source><year>2004</year><publisher-name>Springer Science</publisher-name><pub-id pub-id-type="doi">10.1007/978-1-4757-3990-9</pub-id></nlm-citation></ref><ref id="ref37"><label>37</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Jaeger</surname><given-names>TF</given-names> </name></person-group><article-title>Categorical data analysis: away from ANOVAs (transformation or not) and towards logit mixed models</article-title><source>J Mem Lang</source><year>2008</year><month>11</month><volume>59</volume><issue>4</issue><fpage>434</fpage><lpage>446</lpage><pub-id pub-id-type="doi">10.1016/j.jml.2007.11.007</pub-id><pub-id pub-id-type="medline">19884961</pub-id></nlm-citation></ref><ref id="ref38"><label>38</label><nlm-citation citation-type="book"><person-group person-group-type="author"><name name-style="western"><surname>Anderson</surname><given-names>LW</given-names> </name><name name-style="western"><surname>Krathwohl</surname><given-names>DR</given-names> </name></person-group><source>A Taxonomy for Learning, Teaching, and Assessing: A Revision of Bloom&#x2019;s Taxonomy of Educational Objectives: Complete Edition</source><year>2001</year><publisher-name>Addison Wesley Longman, Inc</publisher-name><pub-id pub-id-type="other">9780321084057</pub-id></nlm-citation></ref><ref id="ref39"><label>39</label><nlm-citation citation-type="book"><person-group person-group-type="author"><name name-style="western"><surname>Case</surname><given-names>SM</given-names> </name><name name-style="western"><surname>Swanson</surname><given-names>DB</given-names> </name></person-group><source>Constructing Written Test Questions for the Basic and Clinical Sciences</source><year>1998</year><access-date>2026-10-29</access-date><publisher-name>National Board of Medical Examiners Philadelphia</publisher-name><comment><ext-link ext-link-type="uri" xlink:href="https://health.uconn.edu/faculty-development/wp-content/uploads/sites/69/2017/06/constructing_written_test_questions.pdf">https://health.uconn.edu/faculty-development/wp-content/uploads/sites/69/2017/06/constructing_written_test_questions.pdf</ext-link></comment></nlm-citation></ref><ref id="ref40"><label>40</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Wass</surname><given-names>V</given-names> </name><name name-style="western"><surname>Van der Vleuten</surname><given-names>C</given-names> </name><name name-style="western"><surname>Shatzer</surname><given-names>J</given-names> </name><name name-style="western"><surname>Jones</surname><given-names>R</given-names> </name></person-group><article-title>Assessment of clinical competence</article-title><source>Lancet</source><year>2001</year><month>03</month><volume>357</volume><issue>9260</issue><fpage>945</fpage><lpage>949</lpage><pub-id pub-id-type="doi">10.1016/S0140-6736(00)04221-5</pub-id><pub-id pub-id-type="medline">11289364</pub-id></nlm-citation></ref><ref id="ref41"><label>41</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Flower</surname><given-names>L</given-names> </name><name name-style="western"><surname>Hayes</surname><given-names>JR</given-names> </name></person-group><article-title>A cognitive process theory of writing</article-title><source>Coll Compos Commun</source><year>1981</year><month>12</month><day>1</day><volume>32</volume><issue>4</issue><fpage>365</fpage><lpage>387</lpage><pub-id pub-id-type="doi">10.2307/356600</pub-id></nlm-citation></ref><ref id="ref42"><label>42</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Valmeekam</surname><given-names>K</given-names> </name><name name-style="western"><surname>Marquez</surname><given-names>M</given-names> </name><name name-style="western"><surname>Sreedharan</surname><given-names>S</given-names> </name><name name-style="western"><surname>Kambhampati</surname><given-names>S</given-names> </name></person-group><article-title>On the planning abilities of large language models - a critical investigation</article-title><source>Adv Neural Inf Process Syst</source><year>2023</year><fpage>75993</fpage><lpage>76005</lpage><pub-id pub-id-type="doi">10.52202/075280-3320</pub-id></nlm-citation></ref><ref id="ref43"><label>43</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Offerdahl</surname><given-names>EG</given-names> </name><name name-style="western"><surname>Montplaisir</surname><given-names>L</given-names> </name></person-group><article-title>Student-generated reading questions: diagnosing student thinking with diverse formative assessments</article-title><source>Biochem Mol Biol Educ</source><year>2014</year><volume>42</volume><issue>1</issue><fpage>29</fpage><lpage>38</lpage><pub-id pub-id-type="doi">10.1002/bmb.20757</pub-id><pub-id pub-id-type="medline">24343750</pub-id></nlm-citation></ref><ref id="ref44"><label>44</label><nlm-citation citation-type="web"><article-title>MedEvalArena</article-title><source>GitHub</source><access-date>2026-02-17</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://github.com/bernardolab/medEvalArena">https://github.com/bernardolab/medEvalArena</ext-link></comment></nlm-citation></ref><ref id="ref45"><label>45</label><nlm-citation citation-type="web"><person-group person-group-type="author"><name name-style="western"><surname>Bernardo</surname><given-names>D</given-names> </name></person-group><article-title>MedEvalArena</article-title><source>Zenodo</source><year>2026</year><access-date>2026-10-29</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://doi.org/10.5281/zenodo.20604284">https://doi.org/10.5281/zenodo.20604284</ext-link></comment></nlm-citation></ref></ref-list><app-group><supplementary-material id="app1"><label>Multimedia Appendix 1</label><p>Details of the model roster, specialty distribution, question-generation validity, pairwise comparisons, self-judging bias, model performance, specialty abbreviations, generation costs, and medical accuracy threshold sensitivity analysis.</p><media xlink:href="jmir_v28i1e96885_app1.docx" xlink:title="DOCX File, 597 KB"/></supplementary-material></app-group></back></article>