<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article article-type="research-article" dtd-version="2.0" xmlns:xlink="http://www.w3.org/1999/xlink">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v28i1e91449</article-id>
      <article-id pub-id-type="pmid"/>
      <article-id pub-id-type="doi">10.2196/91449</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Natural Language Processing Supported by Annotation Process Artifacts to Detect Risky Alcohol Use in Electronic Health Records: Algorithm Development and Validation Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Coristine</surname>
            <given-names>Andrew</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Jawa</surname>
            <given-names>Vibhu</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Elbattah</surname>
            <given-names>Mahmoud</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Zeng</surname>
            <given-names>Sihang</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author">
          <name name-style="western">
            <surname>Weber</surname>
            <given-names>Katherine</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1435-7102</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Strayhorn</surname>
            <given-names>Asher</given-names>
          </name>
          <degrees>BS</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0009-0009-8113-6308</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Ju</surname>
            <given-names>Xintong</given-names>
          </name>
          <degrees>MS</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0009-0005-3091-2726</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Mellinger</surname>
            <given-names>Jessica</given-names>
          </name>
          <degrees>MD, MSc</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7364-5035</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Winder</surname>
            <given-names>Gerald Scott</given-names>
          </name>
          <degrees>MD, MSc</degrees>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-2355-7317</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Fernandez</surname>
            <given-names>Anne C</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9129-7318</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Vydiswaran</surname>
            <given-names>VG Vinod</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Department of Learning Health Sciences</institution>
            <institution>Medical School</institution>
            <institution>University of Michigan</institution>
            <addr-line>1111 E. Catherine Street</addr-line>
            <addr-line>Room 209</addr-line>
            <addr-line>Ann Arbor, MI, 48109-2054</addr-line>
            <country>United States</country>
            <fax>1 734 647 3914</fax>
            <phone>1 734 647 1207</phone>
            <email>vgvinodv@umich.edu</email>
          </address>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3122-1936</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Learning Health Sciences</institution>
        <institution>Medical School</institution>
        <institution>University of Michigan</institution>
        <addr-line>Ann Arbor, MI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Psychiatry</institution>
        <institution>Medical School</institution>
        <institution>University of Michigan</institution>
        <addr-line>Ann Arbor, MI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Division of Gastroenterology &amp; Hepatology</institution>
        <institution>Department of Internal Medicine</institution>
        <institution>Henry Ford Hospital</institution>
        <addr-line>Detroit, MI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Department of Psychiatry</institution>
        <institution>Henry Ford Hospital</institution>
        <addr-line>Detroit, MI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>School of Information</institution>
        <institution>University of Michigan</institution>
        <addr-line>Ann Arbor, MI</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: VG Vinod Vydiswaran <email>vgvinodv@umich.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2026</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>20</day>
        <month>11</month>
        <year>2026</year>
      </pub-date>
      <volume>28</volume>
      <elocation-id>e91449</elocation-id>
      <history>
        <date date-type="received">
          <day>14</day>
          <month>1</month>
          <year>2026</year>
        </date>
        <date date-type="rev-request">
          <day>13</day>
          <month>4</month>
          <year>2026</year>
        </date>
        <date date-type="rev-recd">
          <day>23</day>
          <month>9</month>
          <year>2026</year>
        </date>
        <date date-type="accepted">
          <day>24</day>
          <month>9</month>
          <year>2026</year>
        </date>
      </history>
      <copyright-statement>©Katherine Weber, Asher Strayhorn, Xintong Ju, Jessica Mellinger, Gerald Scott Winder, Anne C Fernandez, VG Vinod Vydiswaran. Originally published in the Journal of Medical Internet Research (https://www.jmir.org), 20.11.2026.</copyright-statement>
      <copyright-year>2026</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research (ISSN 1438-8871), is properly cited. The complete bibliographic information, a link to the original publication on https://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://www.jmir.org/2026/1/e91449" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>High-risk alcohol use and alcohol use disorder (AUD) are significant contributors to surgical risk and complications yet remain underidentified in surgical populations. While electronic health records (EHRs) contain relevant information about patients’ alcohol use in clinical notes, manual chart review is time-intensive and impractical at scale. Natural language processing (NLP) offers a path to automated identification, but alcohol-related risk classes are highly imbalanced in clinical populations, making multiclass classification particularly challenging.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aims to develop an automated NLP pipeline for 4-class alcohol risk classification using clinical notes in a surgical population, addressing class imbalance by leveraging intermediate annotation evidence generated during the human labeling process.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>We manually labeled 1000 surgical patients drawn from a large academic health system into ordinal alcohol risk classes (no-alcohol, low-risk, high-risk, and probable-dependence) and a fifth class indicating no mention of alcohol use. We used a structured annotation process with 6 trained annotators and expert-developed guidelines. We developed a multistage NLP pipeline operating on 3 years of clinical notes per patient. First, a rules-based preprocessing stage filtered notes to clinically relevant sections containing alcohol-related content. No-mention cases were excluded from further modeling. Second, a BioClinicalBERT-based multilabel classifier (“auto-annotator”) was fine-tuned to reproduce the annotations created by human labelers at the section level, producing a structured 26-dimensional probability vector for each section. Finally, a bidirectional long short-term memory (LSTM) network classified patients into the 4 risk categories by processing each patient’s ordered sequence of auto-annotation vectors, allowing the model to capture temporally accumulating evidence across clinical encounters.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Human annotators achieved 93.7% first-pass agreement overall (Fleiss κ=0.8891), but agreement was as low as 67.5% for probable-dependence cases. The 4-class LSTM classifier achieved a weighted-average <italic>F</italic><sub>1</sub>-score of 0.86 and a macroaverage <italic>F</italic><sub>1</sub>-score of 0.77 on a held-out validation set of 195 patients. One-vs-rest area under the curve (AUC) scores were 0.97 for no-alcohol, 0.88 for low-risk, 0.90 for probable-dependence, and 0.71 for high-risk. Collapsed to a binary high/low risk classification, the model achieved an accuracy of 0.92 and a macro-<italic>F</italic><sub>1</sub>-score of 0.80. Performance was comparable across biological sex without explicit sex-specific training.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>We used a 2-stage NLP pipeline to identify 4 ordinal classes of alcohol use risk in surgical patients. Four-class alcohol-use classification presents a highly imbalanced problem that constrains the availability of high-importance but rare labels in the dataset. We addressed this data scarcity by using intermediate annotation evidence as well as final labels to enrich the feature space for the classification task. The pipeline was able to separate patient records into 4 strata of alcohol-use risk, with “high-risk” patients being most likely to be misclassified into an adjacent class.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>natural language processing</kwd>
        <kwd>NLP</kwd>
        <kwd>machine learning</kwd>
        <kwd>electronic health record</kwd>
        <kwd>EHR</kwd>
        <kwd>alcohol-related disorder</kwd>
        <kwd>neural network</kwd>
        <kwd>computer</kwd>
        <kwd>risk assessment</kwd>
        <kwd>substance use disorder</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background and Significance</title>
        <p>Alcohol is the most widely used substance in the United States, and alcohol use disorder (AUD) is the most common type of substance use disorder in the nation [<xref ref-type="bibr" rid="ref1">1</xref>]. Risky alcohol use (ie, consumption of alcohol above recommended daily, weekly, or per-occasion amounts) [<xref ref-type="bibr" rid="ref2">2</xref>] is also common and increasing in some subpopulations [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>]. Both risky alcohol use and AUD pose substantial surgical risk and affect a large proportion of surgical patients [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref6">6</xref>]. Risky alcohol use and AUD are linked to an increased risk of surgical complications, longer hospital stays, increased costs, and even death [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref15">15</xref>]. However, patients with these conditions remain alarmingly underidentified in surgical care [<xref ref-type="bibr" rid="ref16">16</xref>-<xref ref-type="bibr" rid="ref19">19</xref>].</p>
        <p>Proper identification of surgical patients with risky alcohol use and AUD is needed to deliver appropriate surgical care, timely interventions, and inpatient alcohol withdrawal prophylaxis when necessary [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>]. Implementation of interventions is hindered by suboptimal alcohol screening in real-world care. Past research indicates that electronic health records (EHRs) contain relevant alcohol information in clinical notes and other text that could yield insight into the presence of risky alcohol use and AUD even when formal diagnoses are lacking [<xref ref-type="bibr" rid="ref22">22</xref>-<xref ref-type="bibr" rid="ref24">24</xref>]. Access to this type of text information traditionally requires time-intensive manual chart review. Natural language processing (NLP) tools can make this information more accessible to clinicians and researchers earlier in the preoperative time course by using existing information that can supplement formal alcohol screening procedures. NLP is a core information extraction technique in health care because key information, such as clinical notes, social media health discussions, and clinical guidelines, is in the form of narrative text [<xref ref-type="bibr" rid="ref25">25</xref>]. Because the definition of risky alcohol use extends beyond the specific criteria for AUD and because AUD is generally underdiagnosed, NLP approaches can examine the unstructured content of the clinical record for text-based indicators of alcohol-related issues [<xref ref-type="bibr" rid="ref22">22</xref>,<xref ref-type="bibr" rid="ref26">26</xref>].</p>
        <p>Recent applications of NLP to substance use classification include a convolutional neural network (CNN) for screening for substance misuse (area under the receiver operating characteristic curve [AUROC] 0.97, 95% CI 0.96-0.98) [<xref ref-type="bibr" rid="ref27">27</xref>] and a logistic regression classifier using concept unique identifiers (CUIs) generated by an NLP algorithm (AUROC=0.91, 95% CI 0.89-0.92) [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref29">29</xref>]. These classifiers produce a binary label for substance use disorder (or not), rather than a graded degree of risk. Another study found that only 44% of patients with mentions of AUD in clinical notes also had structured codes indicating these issues, highlighting the need for and potential contribution of text analysis [<xref ref-type="bibr" rid="ref24">24</xref>].</p>
        <p>There are a number of tools for identifying conditions related to substance use in the EHR, including individual, custom-developed algorithms [<xref ref-type="bibr" rid="ref23">23</xref>] and processing using one or more publicly available NLP tools [<xref ref-type="bibr" rid="ref30">30</xref>]. Recent efforts at identifying substance use include a regular-expression and rules-based approach that correctly identified 87% of human-labeled patients with risky alcohol use [<xref ref-type="bibr" rid="ref22">22</xref>] and a combination of embeddings and a time-aware long short-term memory (TLSTM) network to detect evidence of alcohol and substance use disorders in clinical notes (AUROC=0.93) [<xref ref-type="bibr" rid="ref31">31</xref>]. Some approaches use methods that extract computable entities from text, such as cTAKES [<xref ref-type="bibr" rid="ref32">32</xref>], to produce features for machine learning classifiers [<xref ref-type="bibr" rid="ref28">28</xref>]. Other NLP-driven approaches for locating evidence of substance use in notes include tools such as pretrained BERT (Bidirectional Encoder Representations from Transformers) models and universal sentence encoders [<xref ref-type="bibr" rid="ref33">33</xref>,<xref ref-type="bibr" rid="ref34">34</xref>] to infer context and meaning in free text. Several pretrained BERT models are tuned on health-related text corpora, including BioClinicalBERT [<xref ref-type="bibr" rid="ref35">35</xref>], a fine-tuned XLM-RoBERTa approach [<xref ref-type="bibr" rid="ref36">36</xref>], and transformer-based models [<xref ref-type="bibr" rid="ref37">37</xref>]. To extract meaning from a large, unfiltered collection of notes and produce a graded risk label, we assembled a pipeline that combines several of these approaches.</p>
      </sec>
      <sec>
        <title>Objective</title>
        <p>The objective of the study is to develop an automated approach to classify alcohol risk using clinical text from EHRs and NLP. We address the challenge of locating clinically significant classes within a highly imbalanced dataset by leveraging both final labels and intermediate annotation information from the labeling process.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Risky Alcohol Use Class Definitions</title>
        <p>Previous efforts to use NLP to identify risky alcohol use in clinical notes have returned a binary positive/negative prediction. To provide health care professionals with a more nuanced view of a patient’s alcohol risk levels, this study defines 4 ordinal risk categories (detailed annotation guidelines for risk classification criteria are available in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        <list list-type="bullet">
          <list-item>
            <p>No-alcohol: information in the clinical text explicitly indicates that the patient does not consume any alcohol.</p>
          </list-item>
          <list-item>
            <p>Low-risk: clinical text indicates that the patient consumes alcohol but does not exceed nationally recommended alcohol consumption limits (ie, no more than 4 drinks per day/occasion or 14 in a week for men, and no more than 3 drinks per day/occasion or 7 per week for women) [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref38">38</xref>], with no evidence of alcohol-related consequences or other criteria for AUD. This category also includes those who previously consumed alcohol or had AUD but have been sober (have not consumed alcohol) for more than 1 year.</p>
          </list-item>
          <list-item>
            <p>High-risk: the clinical text indicates that the patient exceeds nationally recommended alcohol consumption limits, has a diagnosis of mild or moderate AUD, and/or has evidence of 1 alcohol-related consequence, including specific events (eg, driving while intoxicated) or social, emotional, or family problems related to alcohol use.</p>
          </list-item>
          <list-item>
            <p>Probable-dependence: clinical text indicates that the patient is diagnosed with severe AUD, has specific alcohol-associated medical conditions, has attended treatment for alcohol problems, and/or has multiple alcohol-related consequences.</p>
          </list-item>
        </list>
        <p>A fifth category, “no-mention,” was used to label cases in which alcohol-related information is absent or scarce in the patient’s available records. These cases would eventually be excluded from NLP-driven classification because of insufficient data.</p>
      </sec>
      <sec>
        <title>Data Source</title>
        <p>The study was conducted using data from a cohort of patients who underwent 1 or more surgical procedures and participated in the Michigan Genomic Initiative (MGI) [<xref ref-type="bibr" rid="ref39">39</xref>], a large longitudinal cohort study conducted at a large tertiary academic health system in the Midwestern United States. The data extracted included inpatient and outpatient clinical notes, problem lists, visit summaries and letters, diagnostic codes, laboratory results, medications, and demographic information (including biological sex) for 3 years prior to the incident surgery for each patient.</p>
        <p>The working cohort consisted of MGI patients (n=53,949) who met the inclusion criterion of having an index surgery date between May 29, 2012, and April 17, 2019. Patients with no structured data at all, without any clinical notes from encounters in the 3 years before surgery, or whose combined clinical notes contained fewer than 100 characters were excluded (n=320). From the remaining patients, 1000 patients were sampled for manual classification into the 4 risk classes or “no-mention.” We aimed for a dataset that oversampled the rare cases, targeting 5% probable-dependence and 10% high-risk cases, and performed stratified sampling using keywords. The final dataset contains 4% and 8.8%, respectively. <xref ref-type="table" rid="table1">Table 1</xref> describes the demographic characteristics of the patients in the dataset.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Demographic characteristics of the manually labeled patient cohort used for training and evaluation of an NLPa classifier for risky alcohol use. Patients with elective surgeries between 2012 and 2019 were drawn from the Michigan Genomics Initiative.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="190"/>
            <col width="110"/>
            <col width="0"/>
            <col width="120"/>
            <col width="110"/>
            <col width="0"/>
            <col width="100"/>
            <col width="0"/>
            <col width="100"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="0"/>
            <col width="110"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Characteristic</td>
                <td colspan="2">Overall, n=1000</td>
                <td>No-mention, n=24</td>
                <td colspan="2">No-alcohol, n=247</td>
                <td colspan="2">Low-risk, n=601</td>
                <td colspan="2">High-risk, n=88</td>
                <td colspan="2">Probable-dependence, n=40</td>
                <td colspan="2">Missing responses, n</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="14">
                  <bold>Sex, n (%)</bold>
                </td>
                <td>0</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Female</td>
                <td>483 (48.3)</td>
                <td colspan="2">12 (50)</td>
                <td>147 (59.5)</td>
                <td colspan="2">280 (46.6)</td>
                <td colspan="2">31 (35.2)</td>
                <td colspan="2">13 (32.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Male</td>
                <td>517 (51.7)</td>
                <td colspan="2">12 (50)</td>
                <td>100 (40.5)</td>
                <td colspan="2">321 (53.4)</td>
                <td colspan="2">57 (64.8)</td>
                <td colspan="2">27 (67.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="14">
                  <bold>Race, n (%)</bold>
                </td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>African American</td>
                <td>57 (5.7)</td>
                <td colspan="2">—<sup>b</sup></td>
                <td>13 (5.3)</td>
                <td colspan="2">37 (6.2)</td>
                <td colspan="2">2 (2.3)</td>
                <td colspan="2">5 (12.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>American Indian or Alaska Native</td>
                <td>8 (0.8)</td>
                <td colspan="2">1 (4.2)</td>
                <td>2 (0.8)</td>
                <td colspan="2">4 (0.7)</td>
                <td colspan="2">—</td>
                <td colspan="2">1 (2.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Asian</td>
                <td>13 (1.3)</td>
                <td colspan="2">—</td>
                <td>5 (2)</td>
                <td colspan="2">8 (1.3)</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>White</td>
                <td>893 (89.4)</td>
                <td colspan="2">20 (83.3)</td>
                <td>223 (90.3)</td>
                <td colspan="2">534 (89)</td>
                <td colspan="2">83 (94.3)</td>
                <td colspan="2">33 (82.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Native Hawaiian and other Pacific Islander</td>
                <td>1 (0.1)</td>
                <td colspan="2">—</td>
                <td>—</td>
                <td colspan="2">1 (0.2)</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Patient refused</td>
                <td>1 (0.1)</td>
                <td colspan="2">—</td>
                <td>—</td>
                <td colspan="2">1 (0.2)</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Unknown</td>
                <td>10 (1)</td>
                <td colspan="2">1 (4.2)</td>
                <td>2 (0.8)</td>
                <td colspan="2">7 (1.2)</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>16 (1.6)</td>
                <td colspan="2">2 (8.3)</td>
                <td>2 (0.8)</td>
                <td colspan="2">8 (1.3)</td>
                <td colspan="2">3 (3.4)</td>
                <td colspan="2">1 (2.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="14">
                  <bold>Ethnicity, n (%)</bold>
                </td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Hispanic or Latino</td>
                <td>26 (2.6)</td>
                <td colspan="2">2 (8.3)</td>
                <td>4 (1.6)</td>
                <td colspan="2">14 (2.3)</td>
                <td colspan="2">5 (5.7)</td>
                <td colspan="2">1 (2.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Non-Hispanic or Latino</td>
                <td>950 (95.1)</td>
                <td colspan="2">20 (83.3)</td>
                <td>236 (95.5)</td>
                <td colspan="2">577 (96.2)</td>
                <td colspan="2">81 (92)</td>
                <td colspan="2">36 (90)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Unknown</td>
                <td>21 (2.1)</td>
                <td colspan="2">2 (8.3)</td>
                <td>6 (2.4)</td>
                <td colspan="2">8 (1.3)</td>
                <td colspan="2">2 (2.3)</td>
                <td colspan="2">3 (7.5)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Patient refused</td>
                <td>2 (0.2)</td>
                <td colspan="2">—</td>
                <td>1 (0.4)</td>
                <td colspan="2">1 (0.2)</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="14">
                  <bold>Marital status, n (%)</bold>
                </td>
                <td>51</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Married</td>
                <td>567 (59.7)</td>
                <td colspan="2">16 (69.6)</td>
                <td>128 (55.4)</td>
                <td colspan="2">354 (61.6)</td>
                <td colspan="2">47 (56)</td>
                <td colspan="2">22 (61.1)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Unmarried</td>
                <td>382 (40.3)</td>
                <td colspan="2">7 (30.4)</td>
                <td>103 (44.6)</td>
                <td colspan="2">221 (38.4)</td>
                <td colspan="2">37 (44)</td>
                <td colspan="2">14 (38.9)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="14">
                  <bold>Age (years), mean (SD)</bold>
                </td>
                <td>0</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>At surgery</td>
                <td>53.8 (16.0)</td>
                <td colspan="2">53.4 (20.2)</td>
                <td>53.8 (17.2)</td>
                <td colspan="2">53.4 (15.5)</td>
                <td colspan="2">54.8 (15.8)</td>
                <td colspan="2">56.5 (14.9)</td>
                <td colspan="3">
                  <break/>
                </td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>NLP: natural language processing.</p>
            </fn>
            <fn id="table1fn2">
              <p><sup>b</sup>Not applicable.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Creating the Annotated Dataset</title>
        <p>To facilitate manual classification of risky alcohol use in patients, a detailed annotation guideline document was created (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>). The annotation guidelines included detailed definitions of the 4 classes and types of evidence that annotators should consider and annotate before assigning the risky alcohol use classification label. The document also included instructions on using the brat (brat rapid annotation tool) software tool [<xref ref-type="bibr" rid="ref40">40</xref>] for annotation. The guideline document was developed and iteratively refined by 4 experts in addiction diagnosis and treatment, including 2 psychologists (ACF and FB) and 2 physicians (JLM and GSW). The experts based the annotation guidelines on published standards and research, including (1) national definitions for alcohol-related health risks at specific alcohol quantity and frequency cutoffs for men and women [<xref ref-type="bibr" rid="ref41">41</xref>]; (2) AUD symptom type and timing as defined by the <italic>DSM-5</italic> (<italic>Diagnostic and Statistical Manual of Mental Disorders</italic> [Fifth Edition]) [<xref ref-type="bibr" rid="ref38">38</xref>]; and (3) existing research specific to alcohol and surgical risk, which demonstrates increased surgical morbidity when active heavy drinking and AUD precede surgery, but reduced risk when patients abstain from alcohol use in the weeks and months leading up to surgery [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref42">42</xref>]. Twenty patients’ complete notes were manually annotated by these experts following the annotation guidelines and were used to train annotators.</p>
        <p>Annotation was conducted by 6 annotators, including students enrolled in advanced health-related degree programs. Two randomly selected annotators reviewed each chart in chronological order. They used the brat tool to mark segments of text relevant to alcohol risk classification by selecting 1 of 29 different annotation categories for each text segment (see <xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref> for a list of annotation categories). Of those 29 categories, 3 were never used by annotators and were not carried forward into the analysis. Annotation categories included 2-part indicators expressing the kind of evidence (eg, a mention of the amount of alcohol consumed) and the degree of severity (eg, high-risk). After annotators read the entire chart, they recorded their final risky alcohol use label determination (no-alcohol, low-risk, high-risk, or probable-dependence). If additional but identical alcohol use information was repeated in later clinical notes, annotators were not required to mark each repeated mention after the first.</p>
        <p>When annotators disagreed on a label, they were informed that there was a disagreement, and they conducted a “second-pass” annotation independently (they remained blind to the other annotator’s labels). If the disagreement persisted, the annotators met to resolve it through arbitration. During arbitration, the annotators met with the project manager, who moderated the discussion during which annotators provided one another with their rationales and examples that guided their labeling decisions. In cases where they still could not reach an agreement, a third annotator labeled the notes independently to break ties, and the principal investigators reviewed the final decision.</p>
        <p>The final annotated dataset consisted of 1000 patient charts labeled into 1 of the risk classes, as well as all annotations placed on the notes. Eight hundred patients were sampled using stratified sampling to train the NLP pipeline models, and the remaining 200 patients were held out as the validation set. The 24 “no-mention” patients (19 in training and 5 in validation) were subsequently dropped from the pipeline and not used for training subsequent stages, leaving a training set of 781 patients and a validation set of 195 patients.</p>
      </sec>
      <sec>
        <title>Model Design</title>
        <sec>
          <title>Overview</title>
          <p>To label alcohol use risk, we developed a multistage classification pipeline. The first stage, preprocessing and entity detection, standardizes the text input and splits the patient notes into relevant sections with mentions of alcohol use or alcohol-related problems. The second stage, auto-annotation, analyzes each section and returns a vector of probabilities indicating whether any of the 26 annotation labels apply to that section. The 26 possible labels (as well as the 3 that were never chosen by humans in the labeling process) are included in <xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>. The final stage, classification, combines the array of auto-annotation vectors and emits a single class representing the level of alcohol use risk detected in the patient’s notes. The overall classification workflow is summarized in <xref rid="figure1" ref-type="fig">Figure 1</xref>.</p>
          <fig id="figure1" position="float">
            <label>Figure 1</label>
            <caption>
              <p>Schematic diagram of the multistage risky alcohol use classification workflow. LSTM: long short-term memory.</p>
            </caption>
            <graphic xlink:href="jmir_v28i1e91449_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Preprocessing and Entity Detection</title>
          <p>While patient notes are full of relevant health information, information about alcohol-related behavior and conditions is sparsely distributed throughout the patient notes. We narrowed the overall set of notes (n=64,703) by excluding all irrelevant patient instruction notes, retaining 45,550 notes about the 1000 patients in our dataset. The median length of these notes was 317 words. Next, as documents were imported for processing, we applied regular expressions to strip out templated text that instructed patients to avoid drinking alcohol because of a procedure or medication (eg, “heavy alcohol use may increase the risk of toxicity”). The Python script used to drive our rules is provided in <xref ref-type="supplementary-material" rid="app3">Multimedia Appendix 3</xref>.</p>
          <p>We then applied the medspaCy [<xref ref-type="bibr" rid="ref43">43</xref>] toolkit to identify medical entities and context (relations between entities), applying the entity and context rules from a foundation of keywords and logic developed in Vydiswaran et al [<xref ref-type="bibr" rid="ref22">22</xref>]. These rules included regular expressions, proximity calculations, and numerical analysis to identify text related to alcohol use and alcohol-related disorders. The context relations identified by medspaCy include indications of whether the alcohol-related concept (“entity”) was in the past, related to family members, or negated.</p>
          <p>Next, to filter out irrelevant sections from long documents, we broke the notes into sections (such as “social history” and “assessment and plan”) using medspaCy section detection. This resulted in a total of 200,395 sections with a median length of 49 (IQR 15-137) words per section. Of all the sections, only 8417 contained at least 1 alcohol-related entity.</p>
          <p>We experimented with a variety of ways to represent this information as features for classifiers, but the rare classes were entirely ignored or overfitted. At best, macro-<italic>F</italic><sub>1</sub>-scores were 0.66. We developed the auto-annotation approach to provide our models with richer labeled data as an intermediate step.</p>
        </sec>
        <sec>
          <title>Auto-Annotation</title>
          <p>One of the main challenges of the 4-class classification task is that the classes are highly imbalanced, yet the most significant and valuable classes are those that are most infrequent. Models trained with only the final classification were prone to overfitting if we concentrated on the imbalanced classes. Instead of relying on a single label to train a risky alcohol use classifier based on a limited training set, we trained an “auto-annotator” algorithm to build a new feature set that reproduces the annotations created by human annotators as evidence to support a label. This helped address the limitations of a small training dataset, taking advantage of a much larger volume of annotations in the training data to identify key alcohol-risk features in an individual patient’s record.</p>
          <p>To train the auto-annotator, we built a multilabel classifier using the spaCy textcat_multilabel transformers pipeline. The model was trained on sections associated with training-set patients. A total of 6236 sections contained alcohol-related entities, and 128,886 sections had no annotations. We fine-tuned the BioClinicalBERT [<xref ref-type="bibr" rid="ref35">35</xref>] transformer with a new multilabel classification head to return a probability array for each section indicating the probability that the section should trigger each of the 26 annotation classes. Because BERT models are limited to a maximum input sequence of 512 tokens [<xref ref-type="bibr" rid="ref33">33</xref>], we truncated longer sections to 8000 characters and then to 512 tokens after tokenizing in the spaCy pipeline. Approximately 8% of sections were truncated before auto-annotation.</p>
          <p>To validate the auto-annotator, we asked whether its output was functionally equivalent to human annotations as a feature set for downstream classification. We trained 4 scikit-learn classifiers (random forest, decision tree, support vector classifier [SVC], and k-nearest neighbors [KNN]) using auto-annotator output as features. For the best-performing classifier, SVC, we additionally trained the same model using human annotations as features. If the 2 feature sets produced comparable classification performance on the held-out validation set, we considered the auto-annotator a sufficient proxy for human annotation. SVC was the best-performing of the machine learning algorithms—the classifier returned a macro-<italic>F</italic><sub>1</sub>-score of 0.71 over human annotations and 0.69 over auto-annotations.</p>
        </sec>
        <sec>
          <title>Patient Classification</title>
          <p>Preliminary evaluation of these machine-learning models showed 2 key weaknesses, however. First, although precision was generally good overall (particularly for decision tree), recall for high-risk and probable-dependence cases was poor for the KNN, decision tree, and random forest classifiers (macro-<italic>F</italic><sub>1</sub>-scores of 0.51, 0.59, and 0.55, respectively). SVC achieved higher high-risk recall (0.61) than the selected long short-term memory (LSTM; 0.44) but lower overall performance (macro-<italic>F</italic><sub>1</sub>-scores 0.69 vs 0.77). Additionally, the machine-learning models effectively scored all patients as if they were assessed under the guidelines for male patients.</p>
          <p>To generate a final alcohol risk classification for each patient, we turned to 3 neural network model architectures in the PyTorch [<xref ref-type="bibr" rid="ref44">44</xref>] toolkit—a CNN, a multilayer Elman recurrent neural network (RNN), and a bidirectional LSTM model. When the model was given all sections from a patient’s notes, we found that it triggered false positive labels based on sections that had nothing to do with alcohol consumption (anesthesia notes were especially prone to this). As a result, we only sent sections with alcohol-related entities to the classifier. Each model was fed the matrix of k × 26 output from the auto-annotator, where k denotes the number of alcohol-related sections in the patient’s notes. Patients who had no alcohol-related sections were classified as “no-mention.”</p>
          <p>We experimented with a variety of optimizers, learning rates, and samplers, and chose the model that performed well on both the 4-class classification task and the more general 2-class classification into high-risk and low-risk categories. Training data were partitioned into 4 folds, and the model was trained with cross-validation over these folds. Hyperparameter optimization was performed with Weights &amp; Biases [<xref ref-type="bibr" rid="ref45">45</xref>]. The final model used a learning rate of 0.0094 (search was between 0.001 and 0.01), the Adam optimizer (Adam and stochastic gradient descent [SGD] were tested), and a hidden size of 123 (search between 96 and 172). We evaluated PyTorch’s imbalanced dataset sampler as an alternative approach to class imbalance but found no meaningful improvement over the final model configuration.</p>
          <p>We evaluated the final risky alcohol use classifier using both the weighted-average <italic>F</italic><sub>1</sub>-score and the macroaverage <italic>F</italic><sub>1</sub>-score. We also measured the 2-class performance on <italic>F</italic><sub>1</sub>-score.</p>
          <p>To compare this model with binary classifiers for risky alcohol use, we also evaluated the model by rolling the risk classes up, with no-alcohol and low-risk as a single “low” risk class and high-risk and probable-dependence as a “high” risk class. We also split the validation set into male and female patients and evaluated them separately.</p>
        </sec>
      </sec>
      <sec>
        <title>Ethical Considerations</title>
        <p>This study has been reviewed and approved under human participants research guidelines by an institutional review board (study HUM00204651). The research was conducted under an informed consent waiver because it involves no more than minimal risk to the participants, the waiver will not adversely affect the rights or welfare of the participants, and the research could not be carried out without the waiver, as the starting patient dataset included data on thousands of patients. To protect patient privacy and confidentiality, synthetic identifiers (“file numbers”) were generated for each participant. These file numbers were used as the participant identifier in all structured and unstructured data, and all other identifying structured columns were removed from the dataset. The cross-reference between file numbers and patient ID was kept in secure storage and segregated from storage used in the development and testing of models. All data storage and processing took place on a high-security research cluster designed to align with HIPAA (Health Insurance Portability and Accountability Act) data protection regulations. The text of clinical notes could not be reliably deidentified; access to the notes was restricted to members of the research team with appropriate training for handling sensitive data. No compensation was offered to the participants.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Key Results</title>
        <p>To help us set appropriate expectations for model performance, we measured the frequency with which annotators were able to come to a consensus risk class during the labeling process.</p>
        <p><xref ref-type="table" rid="table2">Table 2</xref> shows that human annotators were able to agree on the first pass 93.7% of the time, with Fleiss κ of 0.8891. After the second pass, the agreement rose to 97.9%, with Fleiss κ of 0.9630. While the overall agreement was substantial and “near perfect,” especially after the second pass, human annotators struggled most with identifying high-risk and probable-dependent cases, with their initial agreement as low as 67.5% for the probable-dependent cases. Even after review, 10% of these patients’ notes (as well as the notes of patients eventually labeled high-risk) were reviewed and arbitrated by a third rater.</p>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Interrater agreement between human annotator dyads for classification of risky alcohol use severity in clinical notes, stratified by class. Annotation was performed on 1000 patient records from the Michigan Genomics Initiative.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="410"/>
            <col width="190"/>
            <col width="190"/>
            <col width="210"/>
            <thead>
              <tr valign="top">
                <td>Label</td>
                <td>First pass agreement, n (%)</td>
                <td>Second pass agreement, n (%)</td>
                <td>Required third-party review, n (%)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Probable-dependence (n=40)</td>
                <td>27 (67.5)</td>
                <td>36 (90)</td>
                <td>4 (10)</td>
              </tr>
              <tr valign="top">
                <td>High-risk (n=88)</td>
                <td>68 (77.3)</td>
                <td>79 (89.8)</td>
                <td>9 (10.2)</td>
              </tr>
              <tr valign="top">
                <td>Low-risk (n=601)</td>
                <td>578 (96.2)</td>
                <td>596 (99.2)</td>
                <td>5 (0.8)</td>
              </tr>
              <tr valign="top">
                <td>No-alcohol (n=247)</td>
                <td>240 (97.2)</td>
                <td>244 (98.8)</td>
                <td>3 (1.2)</td>
              </tr>
              <tr valign="top">
                <td>No-mention (n=24)</td>
                <td>24 (100)</td>
                <td>24 (100)</td>
                <td>0 (0)</td>
              </tr>
              <tr valign="top">
                <td>Total (n=1000)</td>
                <td>937 (93.7)</td>
                <td>979 (97.9)</td>
                <td>21 (2.1)</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
        <p>The auto-annotator produced 8417 annotations for the 976 labeled patients. A total of 24 patients had no annotations because no alcohol-related note sections passed the filtering stage; they were labeled as “no-mention.” We experimented with feature arrays with the original 26 classifications used by the annotators, as well as a condensed array of 5 features corresponding to the probability of the 4 severity suffix classes of the annotation categories (eg, “HighRisk” in “AlcoholAmount-HighRisk”) and a fifth feature indicating whether the “Sober-LowRisk” annotation category was used. We found that the 26-class arrays provided more information to the final classifier. The final auto-annotator output is a 26-dimensional vector for each annotated section for every patient.</p>
        <p><xref ref-type="table" rid="table3">Table 3</xref> contains the classification report for the best-performing LSTM classifier. The weighted-average <italic>F</italic><sub>1</sub>-score for the 4-class risky alcohol use classification model was 0.86, and the 4-class macroaverage <italic>F</italic><sub>1</sub>-score was 0.77. When we created a binary risk classifier (combining the bottom 2 and the top 2 risk categories), the accuracy of the 2-class model was 0.92.</p>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Classification performance (precision, recall, and F1-score) of the selected bidirectional LSTM<sup>a</sup> model for graded risky alcohol use detection from clinical notes, evaluated on a held-out validation set of 195 patients from the Michigan Genomics Initiative who underwent elective surgery, 2012-2019. Classes: no-alcohol, low-risk, high-risk, and probable-dependent.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="430"/>
            <col width="0"/>
            <col width="140"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="140"/>
            <col width="0"/>
            <col width="130"/>
            <thead>
              <tr valign="top">
                <td colspan="3">Modeling approach</td>
                <td colspan="2">Precision</td>
                <td colspan="2">Recall</td>
                <td colspan="2"><italic>F</italic><sub>1</sub>-score</td>
                <td>Support</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="10">
                  <bold>Four-class</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Probable-dependence</td>
                <td colspan="2">0.83</td>
                <td colspan="2">0.62</td>
                <td colspan="2">0.71</td>
                <td colspan="2">8</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>High-risk</td>
                <td colspan="2">0.73</td>
                <td colspan="2">0.44</td>
                <td colspan="2">0.55</td>
                <td colspan="2">18</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Low-risk</td>
                <td colspan="2">0.88</td>
                <td colspan="2">0.94</td>
                <td colspan="2">0.91</td>
                <td colspan="2">120</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>No-alcohol</td>
                <td colspan="2">0.90</td>
                <td colspan="2">0.90</td>
                <td colspan="2">0.90</td>
                <td colspan="2">49</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Accuracy</td>
                <td colspan="2">—<sup>b</sup></td>
                <td colspan="2">—</td>
                <td colspan="2">0.87</td>
                <td colspan="2">195</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Macroaverage</td>
                <td colspan="2">0.83</td>
                <td colspan="2">0.73</td>
                <td colspan="2">0.77</td>
                <td colspan="2">195</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Weighted average</td>
                <td colspan="2">0.87</td>
                <td colspan="2">0.87</td>
                <td colspan="2">0.86</td>
                <td colspan="2">195</td>
              </tr>
              <tr valign="top">
                <td colspan="10">
                  <bold>Two-class</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>High</td>
                <td colspan="2">0.82</td>
                <td colspan="2">0.54</td>
                <td colspan="2">0.65</td>
                <td colspan="2">26</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Low</td>
                <td colspan="2">0.93</td>
                <td colspan="2">0.98</td>
                <td colspan="2">0.96</td>
                <td colspan="2">169</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Accuracy</td>
                <td colspan="2">—</td>
                <td colspan="2">—</td>
                <td colspan="2">0.92</td>
                <td colspan="2">195</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Macroaverage</td>
                <td colspan="2">0.88</td>
                <td colspan="2">0.76</td>
                <td colspan="2">0.80</td>
                <td colspan="2">195</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Weighted average</td>
                <td colspan="2">0.92</td>
                <td colspan="2">0.92</td>
                <td colspan="2">0.92</td>
                <td colspan="2">195</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table3fn1">
              <p><sup>a</sup>LSTM: long short-term memory.</p>
            </fn>
            <fn id="table3fn2">
              <p><sup>b</sup>Not applicable.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>The bidirectional LSTM was the best-performing classifier among those we tested for patient classification. Complete classification reports and confusion matrices for all models reported in this study are provided in <xref ref-type="supplementary-material" rid="app4">Multimedia Appendix 4</xref>.</p>
        <p><xref rid="figure2" ref-type="fig">Figures 2</xref> and <xref rid="figure3" ref-type="fig">3</xref> present one-vs-one (OvO) and one-vs-rest (OvR) multiclass receiver operating characteristic (ROC) curves for the final LSTM classifier. The microaverage and macroaverage OvR area under the curve (AUC) scores were 0.94 and 0.87, respectively. Per-class OvR AUC scores were highest for no-alcohol (0.97) and probable-dependence (0.90), followed by low-risk (0.88) and high-risk (0.71). The OvO macro-average AUC was 0.85. Pairwise OvO analysis revealed that the lowest-performing class boundaries involved high-risk, both against low-risk (AUC=0.70) and against probable-dependence (AUC=0.71), while boundaries between other class pairs were substantially higher, ranging from 0.86 to 0.97. Due to the small number of validation cases in the minority classes, ROC curves for probable-dependence and high-risk should be interpreted with caution, as each step in the curve represents 1 or 2 patients.</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>One-vs-one receiver operating characteristic curves for the 4-class model imputing alcohol-use risk in a population of surgical patients drawn from the Michigan Genomics Initiative, 2012 to 2019. AUC: area under the curve.</p>
          </caption>
          <graphic xlink:href="jmir_v28i1e91449_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>One-vs-rest receiver operating characteristic (ROC) curves for the 4-class model imputing alcohol-use risk in a population of surgical patients drawn from the Michigan Genomics Initiative, 2012 to 2019. AUC: area under the curve.</p>
          </caption>
          <graphic xlink:href="jmir_v28i1e91449_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p><xref rid="figure4" ref-type="fig">Figure 4</xref> shows confusion matrices for the final selected model (“good-sweep-27”), as well as 2 other alternate candidate LSTM models. Although the final candidate models’ performances were very similar (alternative 1 [“daily-sweep-51”]: macro-<italic>F</italic><sub>1</sub>-score 0.71; alternative 2 [“prime-sweep-56”]: macro-<italic>F</italic><sub>1</sub>-score 0.76), we selected our model based on the overall number of correct predictions as well as good off-diagonal performance for the high-risk class. Below these, we show the results for the selected model, split by the participant’s biological sex. Macroaverage <italic>F</italic><sub>1</sub>-score is 0.80 for female patients and 0.74 for male patients.</p>
        <fig id="figure4" position="float">
          <label>Figure 4</label>
          <caption>
            <p>Confusion matrices for 3 finalist models (alternative 1: “daily-sweep-51” F1-score 0.71; alternative 2: “prime-sweep-56” F1-score 0.76; selected model: “good-sweep-27” F1-score 0.77), as well as the final model split by sex over the validation set (n=195).</p>
          </caption>
          <graphic xlink:href="jmir_v28i1e91449_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Alternative Model Architecture Comparisons</title>
        <p>We also considered alternative approaches to building a patient classifier based on auto-annotator output. The CNN and RNN architectures did not approach the <italic>F</italic><sub>1</sub>-scores of the LSTM model on the validation set. The overall macroaverage <italic>F</italic><sub>1</sub>-scores of the CNN and RNN models were 0.55 and 0.63, respectively, in contrast to the LSTM’s score of 0.77. We believe the LSTM’s design, which assumes sequenced data, was able to better interpret evidence from auto-annotations as patient alcohol use changed over time.</p>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>Risky alcohol use and AUD are underdiagnosed in structured EHR data but are more frequently noted in clinical text. This important text information is not easily discovered by busy clinicians within a large body of clinical notes. However, this information is clinically important to patients’ surgical planning, recovery, health, and safety. Classifying risky alcohol use is an inherently imbalanced learning problem in which the classes of primary interest are a small minority of the cases available for training. It is difficult to amass enough training data to correctly infer its presence in notes. Our multistage pipeline uses the large volume of intermediate annotations produced by human labelers to enrich the dataset with better features that assist a final classifier.</p>
        <p>Four-class labeling is complicated by sex-specific clinical rules and by the small number of examples of high-risk alcohol use within the dataset. When this model’s output is rolled up into a binary high/low risk classification, the pipeline identifies patients with risky alcohol use or AUD with 92% accuracy, high-risk sensitivity 0.54 (95% CI 0.35-0.71), and specificity 0.98 (95% CI 0.95-0.99; CIs calculated via the Wilson method [<xref ref-type="bibr" rid="ref46">46</xref>,<xref ref-type="bibr" rid="ref47">47</xref>]). Because the high-risk group comprises 13% of the validation set, accuracy is dominated by correct classification of low-risk patients. While the 4-class classification presented in this paper provides a more nuanced understanding of patients with AUD, the binary high/low risk classification is helpful when the classifier is intended to provide clinicians with a chart-based indicator for risky alcohol use prior to surgery.</p>
        <p>Even with 1000 labeled patients, there are very few examples of high-risk and probable-dependent cases—even fewer when the dataset is split by patient sex. While developing this pipeline, we generated feature sets for notes that included a one-hot-encoded array of medspaCy-detected entities, risk scores calculated based on the entities detected, rulesets based on the detected entities, and auto-annotations generated from the severity suffix of annotations. None of these variations was able to support models that approached the performance of our final classifier. Instead of focusing on the final label, we opted to use more-plentiful human-generated evidence from the annotation process to build a richer set of features to aid classification.</p>
        <p>In our model selection process, we opted to maximize the macro-average (vs the weighted average) so the performance on instances from infrequent classes was not lost among those from the more frequent classes. When models being evaluated together had similar macroaverage <italic>F</italic><sub>1</sub>-scores, we prioritized recall over precision in the low-count, high-risk classes, as recall has greater clinical significance. In the clinical context, a false positive would have less clinical risk than a false negative, given that a positive result is intended to lead to additional alcohol screening and a conversation between the clinician and their patient, but a false negative could miss a critical surgical risk factor [<xref ref-type="bibr" rid="ref7">7</xref>,<xref ref-type="bibr" rid="ref15">15</xref>].</p>
        <p>We also split and evaluated the validation set by sex at birth for each model being considered, as we had noted in early trials that models tended to score all patients as if they were male. We had considered training separate models per sex but were concerned about losing even more signal when the rare classes were further subdivided. The final pipeline, a single auto-annotator and a single classification model, performs relatively well for both sexes without needing explicit sex-specific instruction. We believe the information is coming from the auto-annotator features, which we think may have identified a pattern linking pronouns, alcohol quantity references, and the annotation labels.</p>
      </sec>
      <sec>
        <title>Comparison With Prior Work</title>
        <p>Direct comparison between this pipeline and other approaches is difficult because our primary task was extracting 4, rather than 2 (positive and negative), classes of alcohol use risk and because of the diversity of reporting metrics. While we present results for a binary equivalent in <xref ref-type="table" rid="table4">Table 4</xref> for the sake of comparison, our work primarily contributes to the field with its graded, 4-class output and our auto-annotation strategy that provides more information to a classifier in a limited, unbalanced dataset.</p>
        <table-wrap position="float" id="table4">
          <label>Table 4</label>
          <caption>
            <p>Outcomes of NLP<sup>a</sup>-driven experiments that identify risky alcohol use or AUD<sup>b</sup> in clinical notes.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="180"/>
            <col width="180"/>
            <col width="160"/>
            <col width="170"/>
            <col width="310"/>
            <thead>
              <tr valign="top">
                <td>Study</td>
                <td>Task/outcome</td>
                <td>Dataset</td>
                <td>Model type</td>
                <td>Key reported metrics</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>This study</td>
                <td>Four-class alcohol-use risk</td>
                <td>1000 patients; 64,703 notes annotated; and held-out test, n=195 (26 positive)</td>
                <td>BioClinicalBERT + LSTM<sup>c</sup> pipeline</td>
                <td>Four-class, scored as binary: high-risk or probable-dependence vs the rest)—AUROC<sup>d</sup> 0.77 (95% CI 0.66-0.87), sensitivity 0.54, and specificity 0.98</td>
              </tr>
              <tr valign="top">
                <td>Afshar et al [<xref ref-type="bibr" rid="ref23">23</xref>]; trauma AUDIT<sup>e</sup> NLP</td>
                <td>Binary alcohol misuse vs no misuse</td>
                <td>1422 patients; 91,045 notes</td>
                <td>ML<sup>f</sup> NLP classifier</td>
                <td>AUROC=0.78 (95% CI 0.72-0.85), sensitivity 0.56, specificity 0.89, and rule-based keyword sensitivity of 0.18</td>
              </tr>
              <tr valign="top">
                <td>To et al [<xref ref-type="bibr" rid="ref29">29</xref>]; hospitalized validation</td>
                <td>Alcohol misuse per NIAAA<sup>g</sup></td>
                <td>1000 hospitalized patients</td>
                <td>Prior NLP classifier</td>
                <td>AUROC=0.91 (95% CI 0.90-0.93), sensitivity 0.88, specificity 0.78, positive predictive value 0.85, and negative predictive value 0.82</td>
              </tr>
              <tr valign="top">
                <td>Lin et al [<xref ref-type="bibr" rid="ref28">28</xref>]; external validation unhealthy alcohol use</td>
                <td>Unhealthy alcohol use (AUDIT)</td>
                <td>57,605 encounters</td>
                <td>Prior NLP classifier</td>
                <td>AUROC=0.91, AUPRC<sup>h</sup> 0.56, sensitivity 0.66, specificity 0.98, positive predictive value 0.35, and negative predictive value 0.99 at a cut point of 0.5</td>
              </tr>
              <tr valign="top">
                <td>Vydiswaran et al [<xref ref-type="bibr" rid="ref22">22</xref>]; preoperation risky use, rule‑based NLP</td>
                <td>Risky alcohol use presurgery</td>
                <td>53,629 notes; 100 manually labeled</td>
                <td>Rule-based NLP</td>
                <td>Sensitivity 0.87 vs <italic>ICD</italic><sup>i</sup> 0.29 and specificity 0.84 vs <italic>ICD</italic> 0.90; NLP identified 3 times as many risky users as <italic>ICD</italic></td>
              </tr>
              <tr valign="top">
                <td>Mercurio et al [<xref ref-type="bibr" rid="ref48">48</xref>]; ClinicalBERT AUD<sup>j</sup></td>
                <td>AUD from discharge summaries</td>
                <td>MIMIC‑III<sup>k</sup> (size not reported)</td>
                <td>ClinicalBERT</td>
                <td>Precision 0.90, recall 0.65, <italic>F</italic><sub>1</sub>-score 0.75, AUROC=0.97, AUPRC 0.86, approximately 4% overall underdocumentation, and up to 30% extra AUD vs <italic>ICD</italic> at high confidence</td>
              </tr>
              <tr valign="top">
                <td>Afshar et al [<xref ref-type="bibr" rid="ref27">27</xref>]; SMART‑AI</td>
                <td>Alcohol, opioid, and nonopioid misuse screening</td>
                <td>Training, n=54,915; temporal validation, n=16,917; and external validation, n=1991</td>
                <td>Multilabel CNN<sup>l</sup></td>
                <td>Temporal AUROC=0.97 (all misuse), external AUROC alcohol 0.88, opioid 0.94, and false-negative rates of 0.18-0.19 and false-positive rate of 0.03 across key racial groups</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table4fn1">
              <p><sup>a</sup>NLP: natural language processing.</p>
            </fn>
            <fn id="table4fn2">
              <p><sup>b</sup>AUD: alcohol use disorder.</p>
            </fn>
            <fn id="table4fn3">
              <p><sup>c</sup>LSTM: long short-term memory.</p>
            </fn>
            <fn id="table4fn4">
              <p><sup>d</sup>AUROC: area under the receiver operating characteristic curve.</p>
            </fn>
            <fn id="table4fn5">
              <p><sup>e</sup>AUDIT: Alcohol Use Disorders Identification Test.</p>
            </fn>
            <fn id="table4fn6">
              <p><sup>f</sup>ML: machine learning.</p>
            </fn>
            <fn id="table4fn7">
              <p><sup>g</sup>NIAAA: National Institute on Alcohol Abuse and Alcoholism.</p>
            </fn>
            <fn id="table4fn8">
              <p><sup>h</sup>AUPRC: area under the precision-recall curve.</p>
            </fn>
            <fn id="table4fn9">
              <p><sup>i</sup>ICD: International Classification of Diseases.</p>
            </fn>
            <fn id="table4fn10">
              <p><sup>j</sup>AUD: alcohol use disorder.</p>
            </fn>
            <fn id="table4fn11">
              <p><sup>k</sup>MIMIC-III: Medical Information Mart for Intensive Care III.</p>
            </fn>
            <fn id="table4fn12">
              <p><sup>l</sup>CNN: convolutional neural network.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Error Analysis</title>
        <p>By design, the bidirectional LSTM’s sequential approach mimics the way clinical evidence can grow and change over time. On deeper analysis of the errors made by the model, we noticed that for patients who had recently stopped drinking, their past behavior was often weighted more than a recent period of sobriety. This may be due to a very low number of examples of sobriety in the training data. Additional training and experimentation are needed to test whether biasing the model to favor recent information leads to better models overall.</p>
        <p>This algorithm tends to downgrade high-risk patients to “low-risk,” but this is offset by its much better performance in this aspect than other models we evaluated. The selected model misclassified 2 of the 8 “probable-dependence” validation cases as “low-risk.” We chose to accept this performance trade-off because of the higher likelihood that a “probable-dependent” patient will have other indicators of AUD in the clinical record, such as diagnostic codes or alcohol-related comorbidities.</p>
        <p>The issues in the high-risk class are also apparent in the ROC analysis: high-risk confusion occurs at both the high-low boundary as well as at the high-dependent boundary. We hypothesize that the difficulties between high- and low-risk arise in part from the difference in rules for male and female patients. Although this pipeline is about as performant for each sex, our decision to stay with a single model is probably responsible for some of these errors.</p>
        <p>The model’s difficulties with the more severe classes mirror the challenges experienced by the human annotators. Agreement between annotators was as low as 67.5% for the probable-dependent cases, and even after 1 round of conferring, 10% of the cases that were eventually labeled “probable-dependence” had to be arbitrated by a third annotator. This underscores the challenge of the alcohol risk classification task using clinical text that can be unclear and ambiguous.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>The model has several limitations. First, it was trained exclusively on notes generated during clinical care in one health system. As local dialects, templates, documentation styles, and writing patterns change between institutions and communities, and with the introduction of writing assistants in documenting clinical encounters, the patterns distinguished by the auto-annotator over these notes may need to be adapted periodically and when applied in a different institutional setting. Some model elements cannot be released publicly due to privacy limitations.</p>
        <p>Second, the 2-stage approach introduces the possibility of error accumulation; misclassification in the auto-annotation stage could propagate to the final classifier. Our functional equivalence validation partially addresses this concern, but a systematic error-case analysis was beyond the scope of this study and is identified as future work.</p>
        <p>Third, the auto-annotator was trained without explicit class-balancing techniques such as weighted loss or undersampling of negative sections. Given the approximately 20:1 ratio of unannotated to annotated sections, this represents a limitation of the current implementation. However, the functional equivalence validation, which showed that downstream classification performance was comparable whether trained on auto-annotations or human annotations, provides evidence that the auto-annotator produced useful signal despite this imbalance.</p>
        <p>Additionally, the pipeline assumes that initial alcohol-related entity detection had acceptable recall, as the ruleset used to detect alcohol entities was derived from an algorithm developed in an earlier experiment [<xref ref-type="bibr" rid="ref22">22</xref>]. We note that a coding error (retained in <xref ref-type="supplementary-material" rid="app3">Multimedia Appendix 3</xref> for reference) prevented detection of the tokens “none,” “abstained,” “abstinent,” “wks,” and “quit” in a rule designed to capture a period of sobriety. Rules to capture a period of time since a last drink, or time since a person quit, were affected by failure to detect “wks” and “week.”</p>
        <p>Finally, the performance estimates for the probable-dependent class (n=8 in the validation set) are based on a very small sample and should be interpreted with caution; a change of 1 or 2 classifications meaningfully shifts per-class and macroaverage scores. The OvR AUC of 0.90 for this class may show that the pipeline can detect signals of alcohol dependency.</p>
      </sec>
      <sec>
        <title>Future Directions</title>
        <p>This model is currently being validated in a prospective study of presurgical patients who have provided detailed information about their alcohol intake and AUD symptoms. To determine the applicability of this model in clinical practice, we will also evaluate the use of the model as a predictor of adverse postsurgical outcomes across a large cohort of patients and their clinical notes.</p>
        <p>Large language models have moved to the forefront of many NLP tasks. Our preliminary investigation found that the LLMs available had the following issues: foremost, at the time, no HIPAA-compliant LLM was offered at our institution. The costs of processing gigabytes of clinical notes are also a significant issue: the filtering part of this pipeline would need to stay in place. Finally, we did trials with deidentified data and found that even with careful prompting, the LLM tested was prone to applying creative interpretations of our classification rules. LLMs should be investigated more thoroughly for this task, however; a general-purpose model with strict instructions could be a better solution than a relatively brittle, dedicated pipeline.</p>
      </sec>
      <sec>
        <title>Conclusion</title>
        <p>In this paper, we present a 2-stage NLP pipeline that identifies ordinal classes of risky alcohol use in presurgical patients. To address the scarcity of labeled data for high-risk and high-importance classes, we included a BioClinicalBERT-based auto-annotator that mimics human annotators in identifying key evidence from clinical notes and outputs probabilities across 26 dimensions of alcohol use. These features are then fed into an LSTM model to determine the patient-level alcohol use risk in four categories—no-alcohol, low-risk, high-risk, and probable-dependence. The pipeline was able to separate patient records into 4 strata of alcohol-use risk, with “high-risk” patients being most likely to be misclassified into an adjacent class.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Annotation guidelines.</p>
        <media xlink:href="jmir_v28i1e91449_app1.docx" xlink:title="DOCX File , 6019 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Categories for annotating patient records containing evidence of alcohol use, also serving as category labels for the auto-annotator component of the pipeline.</p>
        <media xlink:href="jmir_v28i1e91449_app2.docx" xlink:title="DOCX File , 16 KB"/>
      </supplementary-material>
      <supplementary-material id="app3">
        <label>Multimedia Appendix 3</label>
        <p>Python script containing rules for identifying and preprocessing alcohol-related information in clinical notes.</p>
        <media xlink:href="jmir_v28i1e91449_app3.docx" xlink:title="DOCX File , 21 KB"/>
      </supplementary-material>
      <supplementary-material id="app4">
        <label>Multimedia Appendix 4</label>
        <p>Raw analysis output from the pipeline that applies rules, auto-annotation, and a long short-term memory (LSTM) classifier to patient notes to detect risky alcohol use.</p>
        <media xlink:href="jmir_v28i1e91449_app4.docx" xlink:title="DOCX File , 22 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AUC</term>
          <def>
            <p>area under the curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">AUD</term>
          <def>
            <p>alcohol use disorder</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">AUROC</term>
          <def>
            <p>area under the receiver operating characteristic curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">BERT</term>
          <def>
            <p>Bidirectional Encoder Representations from Transformers</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">CNN</term>
          <def>
            <p>convolutional neural network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">CUI</term>
          <def>
            <p>concept unique identifier</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">DSM-5</term>
          <def>
            <p>Diagnostic and Statistical Manual of Mental Disorders (Fifth Edition)</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">EHR</term>
          <def>
            <p>electronic health record</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">HIPAA</term>
          <def>
            <p>Health Insurance Portability and Accountability Act</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb10">KNN</term>
          <def>
            <p>k-nearest neighbors</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb11">LSTM</term>
          <def>
            <p>long short-term memory</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb12">MGI</term>
          <def>
            <p>Michigan Genomic Initiative</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb13">NLP</term>
          <def>
            <p>natural language processing</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb14">OvO</term>
          <def>
            <p>one-vs-one</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb15">OvR</term>
          <def>
            <p>one-vs-rest</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb16">RNN</term>
          <def>
            <p>recurrent neural network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb17">ROC</term>
          <def>
            <p>receiver operating characteristic</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb18">SGD</term>
          <def>
            <p>stochastic gradient descent</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb19">SVC</term>
          <def>
            <p>support vector classifier</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb20">TLSTM</term>
          <def>
            <p>time-aware long short-term memory</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <notes>
      <title>Data Availability</title>
      <p>The datasets analyzed and models generated are not publicly available due to the presence of personally identifying information in the text that was processed but are available from the corresponding author on reasonable request.</p>
    </notes>
    <notes>
      <title>Funding</title>
      <p>Research reported in this publication was supported by the National Institute on Alcohol Abuse and Alcoholism of the National Institutes of Health (award number R33AA028315). The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.</p>
    </notes>
    <fn-group>
      <fn fn-type="con">
        <p>VGVV and ACF are joint senior authors of the study and principal investigators of the grant that funded this research. VGVV and ACF designed the study protocol, developed annotation guidelines, and directed the annotation process. KW developed the analysis pipeline and deep learning models and analyzed the results. AS helped with data processing, developing preliminary models, and annotations. XJ helped with data analysis and discussion. JLM and GSW advised on annotation guidelines and validating the approaches for clinical significance. ACF led the overall project management and alcohol clinical expertise, and VGVV led the model development efforts. KW wrote the first draft of the manuscript, and all authors contributed to revising and editing the manuscript.</p>
      </fn>
      <fn fn-type="conflict">
        <p>GSW consults for GlaxoSmithKline and Novo Nordisk. JM consults for GlaxoSmithKline, Novo Nordisk, and Boehringer Ingelheim. No other authors have any conflicts of interest to disclose at this time.</p>
      </fn>
    </fn-group>
    <notes>
      <sec>
        <title>Generative AI Use</title>
        <p>The authors declare that generative AI was not used in the preparation of this manuscript.</p>
      </sec>
    </notes>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
          <article-title>Key Substance Use and Mental Health Indicators in the United States: Results From the 2016 National Survey on Drug Use and Health</article-title>
          <source>Substance Abuse and Mental Health Services Administration (SAMHSA)</source>
          <access-date>2023-03-13</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.samhsa.gov/data/report/key-substance-use-and-mental-health-indicators-united-states-results-2016-national-survey">https://www.samhsa.gov/data/report/key-substance-use-and-mental-health-indicators-united-states-results-2016-national-survey</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="web">
          <article-title>Drinking Levels Defined</article-title>
          <source>National Institute on Alcohol Abuse and Alcoholism</source>
          <access-date>2024-02-21</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.niaaa.nih.gov/alcohol-health/overview-alcohol-consumption/moderate-binge-drinking">https://www.niaaa.nih.gov/alcohol-health/overview-alcohol-consumption/moderate-binge-drinking</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Grant</surname>
              <given-names>BF</given-names>
            </name>
          </person-group>
          <article-title>Prevalence and correlates of alcohol use and DSM-IV alcohol dependence in the United States: results of the National Longitudinal Alcohol Epidemiologic Survey</article-title>
          <source>J Stud Alcohol</source>
          <year>1997</year>
          <volume>58</volume>
          <issue>5</issue>
          <fpage>464</fpage>
          <lpage>473</lpage>
          <pub-id pub-id-type="doi">10.15288/jsa.1997.58.464</pub-id>
          <pub-id pub-id-type="medline">9273910</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lechner</surname>
              <given-names>WV</given-names>
            </name>
            <name name-style="western">
              <surname>Sidhu</surname>
              <given-names>NK</given-names>
            </name>
            <name name-style="western">
              <surname>Jin</surname>
              <given-names>JT</given-names>
            </name>
            <name name-style="western">
              <surname>Kittaneh</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Laurene</surname>
              <given-names>KR</given-names>
            </name>
            <name name-style="western">
              <surname>Kenne</surname>
              <given-names>DR</given-names>
            </name>
          </person-group>
          <article-title>Increases in risky drinking during the COVID-19 pandemic assessed via longitudinal cohort design: associations with racial tensions, financial distress, psychological distress and virus-related fears</article-title>
          <source>Alcohol Alcohol</source>
          <year>2021</year>
          <volume>56</volume>
          <issue>6</issue>
          <fpage>702</fpage>
          <lpage>707</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/33765131"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/alcalc/agab019</pub-id>
          <pub-id pub-id-type="medline">33765131</pub-id>
          <pub-id pub-id-type="pii">6188473</pub-id>
          <pub-id pub-id-type="pmcid">PMC8083654</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fernandez</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Waljee</surname>
              <given-names>JF</given-names>
            </name>
            <name name-style="western">
              <surname>Gunaseelan</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Brummett</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Englesbe</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Bicket</surname>
              <given-names>MC</given-names>
            </name>
          </person-group>
          <article-title>Prevalence of unhealthy substance use and associated characteristics among patients presenting for surgery</article-title>
          <source>Ann Surg</source>
          <year>2023</year>
          <volume>278</volume>
          <issue>4</issue>
          <fpage>e740</fpage>
          <lpage>e744</lpage>
          <pub-id pub-id-type="doi">10.1097/SLA.0000000000005767</pub-id>
          <pub-id pub-id-type="medline">36538617</pub-id>
          <pub-id pub-id-type="pii">00000658-202310000-00031</pub-id>
          <pub-id pub-id-type="pmcid">PMC10205913</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>AHS</given-names>
            </name>
            <name name-style="western">
              <surname>Frey</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Debenedetti</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bradley</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Alcohol Misuse Prevalence and Associations with Post-Operative Complications in US Surgical Patients: A Review</article-title>
          <source>Open J Surg</source>
          <year>2008</year>
          <volume>2</volume>
          <fpage>50</fpage>
          <lpage>58</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.benthamopenarchives.com/contents/pdf/TOSJ/TOSJ-2-50.pdf"/>
          </comment>
          <pub-id pub-id-type="doi">10.2174/1874300500802010050</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Eliasen</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Grønkjær</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Skov-Ettrup</surname>
              <given-names>LS</given-names>
            </name>
            <name name-style="western">
              <surname>Mikkelsen</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Becker</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Tolstrup</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Flensborg-Madsen</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Preoperative alcohol consumption and postoperative complications: a systematic review and meta-analysis</article-title>
          <source>Ann Surg</source>
          <year>2013</year>
          <volume>258</volume>
          <issue>6</issue>
          <fpage>930</fpage>
          <lpage>942</lpage>
          <pub-id pub-id-type="doi">10.1097/SLA.0b013e3182988d59</pub-id>
          <pub-id pub-id-type="medline">23732268</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>AHS</given-names>
            </name>
            <name name-style="western">
              <surname>Reeder</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Ellerbe</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Bradley</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Rubinsky</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Giori</surname>
              <given-names>NJ</given-names>
            </name>
          </person-group>
          <article-title>Preoperative alcohol screening scores: association with complications in men undergoing total joint arthroplasty</article-title>
          <source>J Bone Joint Surg Am</source>
          <year>2011</year>
          <volume>93</volume>
          <issue>4</issue>
          <fpage>321</fpage>
          <lpage>327</lpage>
          <pub-id pub-id-type="doi">10.2106/JBJS.I.01560</pub-id>
          <pub-id pub-id-type="medline">21325583</pub-id>
          <pub-id pub-id-type="pii">93/4/321</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bradley</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Rubinsky</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Sun</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Bryson</surname>
              <given-names>CL</given-names>
            </name>
            <name name-style="western">
              <surname>Bishop</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Blough</surname>
              <given-names>DK</given-names>
            </name>
            <name name-style="western">
              <surname>Henderson</surname>
              <given-names>WG</given-names>
            </name>
            <name name-style="western">
              <surname>Maynard</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Hawn</surname>
              <given-names>MT</given-names>
            </name>
            <name name-style="western">
              <surname>Tønnesen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Hughes</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Beste</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>AHS</given-names>
            </name>
            <name name-style="western">
              <surname>Hawkins</surname>
              <given-names>EJ</given-names>
            </name>
            <name name-style="western">
              <surname>Houston</surname>
              <given-names>TK</given-names>
            </name>
            <name name-style="western">
              <surname>Kivlahan</surname>
              <given-names>DR</given-names>
            </name>
          </person-group>
          <article-title>Alcohol screening and risk of postoperative complications in male VA patients undergoing major non-cardiac surgery</article-title>
          <source>J Gen Intern Med</source>
          <year>2011</year>
          <volume>26</volume>
          <issue>2</issue>
          <fpage>162</fpage>
          <lpage>169</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/20878363"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s11606-010-1475-x</pub-id>
          <pub-id pub-id-type="medline">20878363</pub-id>
          <pub-id pub-id-type="pmcid">PMC3019325</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rubinsky</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Bishop</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Maynard</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Henderson</surname>
              <given-names>WG</given-names>
            </name>
            <name name-style="western">
              <surname>Hawn</surname>
              <given-names>MT</given-names>
            </name>
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Beste</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Tønnesen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Bradley</surname>
              <given-names>KA</given-names>
            </name>
          </person-group>
          <article-title>Postoperative risks associated with alcohol screening depend on documented drinking at the time of surgery</article-title>
          <source>Drug Alcohol Depend</source>
          <year>2013</year>
          <volume>132</volume>
          <issue>3</issue>
          <fpage>521</fpage>
          <lpage>527</lpage>
          <pub-id pub-id-type="doi">10.1016/j.drugalcdep.2013.03.022</pub-id>
          <pub-id pub-id-type="medline">23683792</pub-id>
          <pub-id pub-id-type="pii">S0376-8716(13)00119-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tønnesen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Petersen</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Højgaard</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Stokholm</surname>
              <given-names>KH</given-names>
            </name>
            <name name-style="western">
              <surname>Nielsen</surname>
              <given-names>H J</given-names>
            </name>
            <name name-style="western">
              <surname>Knigge</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Kehlet</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Postoperative morbidity among symptom-free alcohol misusers</article-title>
          <source>Lancet</source>
          <year>1992</year>
          <volume>340</volume>
          <issue>8815</issue>
          <fpage>334</fpage>
          <lpage>337</lpage>
          <pub-id pub-id-type="doi">10.1016/0140-6736(92)91405-w</pub-id>
          <pub-id pub-id-type="medline">1353805</pub-id>
          <pub-id pub-id-type="pii">0140-6736(92)91405-W</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Oppedal</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Møller</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Pedersen</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Tønnesen</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Preoperative alcohol cessation prior to elective surgery</article-title>
          <source>Cochrane Database Syst Rev</source>
          <year>2012</year>
          <issue>7</issue>
          <fpage>CD008343</fpage>
          <pub-id pub-id-type="doi">10.1002/14651858.CD008343.pub2</pub-id>
          <pub-id pub-id-type="medline">22786514</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lowery</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Yong</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Joyce</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Kovacs</surname>
              <given-names>EJ</given-names>
            </name>
          </person-group>
          <article-title>Recent alcohol use prolongs hospital length of stay following lung transplant</article-title>
          <source>Clin Transplant</source>
          <year>2018</year>
          <volume>32</volume>
          <issue>6</issue>
          <fpage>e13250</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/29620796"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/ctr.13250</pub-id>
          <pub-id pub-id-type="medline">29620796</pub-id>
          <pub-id pub-id-type="pmcid">PMC6023739</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Spies</surname>
              <given-names>CD</given-names>
            </name>
            <name name-style="western">
              <surname>Nordmann</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Brummer</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Marks</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Conrad</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Berger</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Runkel</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Neumann</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Müller</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Rommelspacher</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Specht</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hannemann</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Striebel</surname>
              <given-names>HW</given-names>
            </name>
            <name name-style="western">
              <surname>Schaffartzik</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Intensive care unit stay is prolonged in chronic alcoholic men following tumor resection of the upper digestive tract</article-title>
          <source>Acta Anaesthesiol Scand</source>
          <year>1996</year>
          <volume>40</volume>
          <issue>6</issue>
          <fpage>649</fpage>
          <lpage>656</lpage>
          <pub-id pub-id-type="doi">10.1111/j.1399-6576.1996.tb04505.x</pub-id>
          <pub-id pub-id-type="medline">8836256</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fernandez</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Bohnert</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Bicket</surname>
              <given-names>MC</given-names>
            </name>
            <name name-style="western">
              <surname>Weng</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Englesbe</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Adverse surgical outcomes linked to co-occurring smoking and risky alcohol use among general surgery patients</article-title>
          <source>Ann Surg</source>
          <year>2023</year>
          <volume>278</volume>
          <issue>2</issue>
          <fpage>201</fpage>
          <lpage>207</lpage>
          <pub-id pub-id-type="doi">10.1097/SLA.0000000000005735</pub-id>
          <pub-id pub-id-type="medline">36268706</pub-id>
          <pub-id pub-id-type="pii">00000658-202308000-00009</pub-id>
          <pub-id pub-id-type="pmcid">PMC10119331</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shourie</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Conigrave</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Proude</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>JE</given-names>
            </name>
            <name name-style="western">
              <surname>Wutzke</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>PS</given-names>
            </name>
          </person-group>
          <article-title>Pre-operative screening for excessive alcohol consumption among patients scheduled for elective surgery</article-title>
          <source>Drug Alcohol Rev</source>
          <year>2007</year>
          <volume>26</volume>
          <issue>2</issue>
          <fpage>119</fpage>
          <lpage>125</lpage>
          <pub-id pub-id-type="doi">10.1080/09595230601146595</pub-id>
          <pub-id pub-id-type="medline">17364846</pub-id>
          <pub-id pub-id-type="pii">771044233</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fernandez</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Alcohol-Related Clinical Practices in Surgical Healthcare: Qualitative Findings From Patients and Healthcare Providers</article-title>
          <source>Stanford University</source>
          <year>2019</year>
          <month>05</month>
          <day>06</day>
          <access-date>2026-10-06</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://med.stanford.edu/content/dam/sm/s-spire/documents/WIP/WIP_5.6.19_Alcohol-Use-and-Surgical-Health-Optimization_Fernandez.pdf">https://med.stanford.edu/content/dam/sm/s-spire/documents/WIP/WIP_5.6.19_Alcohol-Use-and-Surgical-Health-Optimization_Fernandez.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cooley</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bicket</surname>
              <given-names>MC</given-names>
            </name>
            <name name-style="western">
              <surname>Mohammed</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Lai</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Evilsizer</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Brummett</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Waljee</surname>
              <given-names>JF</given-names>
            </name>
          </person-group>
          <article-title>Worth the risk? Standardized screening to identify substance use among patients before surgery</article-title>
          <source>Ann Surg</source>
          <year>2025</year>
          <volume>282</volume>
          <issue>6</issue>
          <fpage>991</fpage>
          <lpage>997</lpage>
          <pub-id pub-id-type="doi">10.1097/SLA.0000000000006358</pub-id>
          <pub-id pub-id-type="medline">38814073</pub-id>
          <pub-id pub-id-type="pii">00000658-990000000-00907</pub-id>
          <pub-id pub-id-type="pmcid">PMC12594147</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kip</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Neumann</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Jugel</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Kleinwaechter</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Weiss-Gerlach</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Guill</surname>
              <given-names>MM</given-names>
            </name>
            <name name-style="western">
              <surname>Spies</surname>
              <given-names>CD</given-names>
            </name>
          </person-group>
          <article-title>New strategies to detect alcohol use disorders in the preoperative assessment clinic of a German university hospital</article-title>
          <source>Anesthesiology</source>
          <year>2008</year>
          <volume>109</volume>
          <issue>2</issue>
          <fpage>171</fpage>
          <lpage>179</lpage>
          <pub-id pub-id-type="doi">10.1097/ALN.0b013e31817f5be3</pub-id>
          <pub-id pub-id-type="medline">18648225</pub-id>
          <pub-id pub-id-type="pii">00000542-200808000-00005</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chiang</surname>
              <given-names>PP</given-names>
            </name>
          </person-group>
          <article-title>Perioperative management of the alcohol-dependent patient</article-title>
          <source>Am Fam Physician</source>
          <year>1995</year>
          <volume>52</volume>
          <issue>8</issue>
          <fpage>2267</fpage>
          <lpage>2273</lpage>
          <pub-id pub-id-type="medline">7484720</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tønnesen</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Møller</surname>
              <given-names>AM</given-names>
            </name>
          </person-group>
          <article-title>The smoking and drinking patient</article-title>
          <source>Ugeskr Laeger</source>
          <year>2006</year>
          <volume>168</volume>
          <issue>49</issue>
          <fpage>4293</fpage>
          <lpage>4296</lpage>
          <pub-id pub-id-type="medline">17164055</pub-id>
          <pub-id pub-id-type="pii">VP48391</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vydiswaran</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Strayhorn</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Weber</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Stevens</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mellinger</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Winder</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Fernandez</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Automated-detection of risky alcohol use prior to surgery using natural language processing</article-title>
          <source>Alcohol Clin Exp Res (Hoboken)</source>
          <year>2024</year>
          <month>01</month>
          <volume>48</volume>
          <issue>1</issue>
          <fpage>153</fpage>
          <lpage>163</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://hdl.handle.net/2027.42/192257"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/acer.15222</pub-id>
          <pub-id pub-id-type="medline">38189663</pub-id>
          <pub-id pub-id-type="pmcid">PMC10783530</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afshar</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Phillips</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Karnik</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Mueller</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>To</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gonzalez</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Price</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Cooper</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Joyce</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dligach</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Natural language processing and machine learning to identify alcohol misuse from the electronic health record in trauma patients: development and internal validation</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2019</year>
          <volume>26</volume>
          <issue>3</issue>
          <fpage>254</fpage>
          <lpage>261</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/30602031"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/jamia/ocy166</pub-id>
          <pub-id pub-id-type="medline">30602031</pub-id>
          <pub-id pub-id-type="pii">5271068</pub-id>
          <pub-id pub-id-type="pmcid">PMC6657384</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Garcia-Webb</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>An analysis of free-text alcohol use documentation in the electronic health record: early findings and implications</article-title>
          <source>Appl Clin Inform</source>
          <year>2014</year>
          <volume>5</volume>
          <issue>2</issue>
          <fpage>402</fpage>
          <lpage>415</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/25024757"/>
          </comment>
          <pub-id pub-id-type="doi">10.4338/ACI-2013-12-RA-0101</pub-id>
          <pub-id pub-id-type="medline">25024757</pub-id>
          <pub-id pub-id-type="pmcid">PMC4081744</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hao</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Liang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Weng</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Health natural language processing: methodology development and applications</article-title>
          <source>JMIR Med Inform</source>
          <year>2021</year>
          <volume>9</volume>
          <issue>10</issue>
          <fpage>e23898</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2021/10/e23898/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/23898</pub-id>
          <pub-id pub-id-type="medline">34673533</pub-id>
          <pub-id pub-id-type="pii">v9i10e23898</pub-id>
          <pub-id pub-id-type="pmcid">PMC8569540</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Liao</surname>
              <given-names>KP</given-names>
            </name>
            <name name-style="western">
              <surname>Cai</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Savova</surname>
              <given-names>GK</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>SN</given-names>
            </name>
            <name name-style="western">
              <surname>Karlson</surname>
              <given-names>EW</given-names>
            </name>
            <name name-style="western">
              <surname>Ananthakrishnan</surname>
              <given-names>AN</given-names>
            </name>
            <name name-style="western">
              <surname>Gainer</surname>
              <given-names>VS</given-names>
            </name>
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>SY</given-names>
            </name>
            <name name-style="western">
              <surname>Xia</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Szolovits</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Churchill</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kohane</surname>
              <given-names>I</given-names>
            </name>
          </person-group>
          <article-title>Development of phenotype algorithms using electronic medical records and incorporating natural language processing</article-title>
          <source>BMJ</source>
          <year>2015</year>
          <volume>350</volume>
          <fpage>h1885</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/25911572"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmj.h1885</pub-id>
          <pub-id pub-id-type="medline">25911572</pub-id>
          <pub-id pub-id-type="pmcid">PMC4707569</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afshar</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Dligach</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Oguss</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Chhabra</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Markossian</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Joyce</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Churpek</surname>
              <given-names>MM</given-names>
            </name>
            <name name-style="western">
              <surname>Karnik</surname>
              <given-names>NS</given-names>
            </name>
          </person-group>
          <article-title>Development and multimodal validation of a substance misuse algorithm for referral to treatment using artificial intelligence (SMART-AI): a retrospective deep learning study</article-title>
          <source>Lancet Digit Health</source>
          <year>2022</year>
          <volume>4</volume>
          <issue>6</issue>
          <fpage>e426</fpage>
          <lpage>e435</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S2589-7500(22)00041-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2589-7500(22)00041-3</pub-id>
          <pub-id pub-id-type="medline">35623797</pub-id>
          <pub-id pub-id-type="pii">S2589-7500(22)00041-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC9159760</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Boley</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Perticone</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Chhabra</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Afshar</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Karnik</surname>
              <given-names>NS</given-names>
            </name>
          </person-group>
          <article-title>External validation of a machine learning classifier to identify unhealthy alcohol use in hospitalized patients</article-title>
          <source>Addiction</source>
          <year>2022</year>
          <volume>117</volume>
          <issue>4</issue>
          <fpage>925</fpage>
          <lpage>933</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/34729829"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/add.15730</pub-id>
          <pub-id pub-id-type="medline">34729829</pub-id>
          <pub-id pub-id-type="pmcid">PMC9296269</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>To</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Karnik</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Joyce</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dligach</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Afshar</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Validation of an alcohol misuse classifier in hospitalized patients</article-title>
          <source>Alcohol</source>
          <year>2020</year>
          <volume>84</volume>
          <fpage>49</fpage>
          <lpage>55</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/31574300"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.alcohol.2019.09.008</pub-id>
          <pub-id pub-id-type="medline">31574300</pub-id>
          <pub-id pub-id-type="pii">S0741-8329(19)30174-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC7101259</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vydiswaran</surname>
              <given-names>VGV</given-names>
            </name>
            <name name-style="western">
              <surname>Strayhorn</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhao</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Robinson</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Agarwal</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Bagazinski</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Essiet</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Iott</surname>
              <given-names>BE</given-names>
            </name>
            <name name-style="western">
              <surname>Joo</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Lu</surname>
              <given-names>JX</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Murali</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sasagawa</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Yuan</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Hybrid bag of approaches to characterize selection criteria for cohort identification</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2019</year>
          <volume>26</volume>
          <issue>11</issue>
          <fpage>1172</fpage>
          <lpage>1180</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/31197354"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/jamia/ocz079</pub-id>
          <pub-id pub-id-type="medline">31197354</pub-id>
          <pub-id pub-id-type="pii">5518584</pub-id>
          <pub-id pub-id-type="pmcid">PMC7647216</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Miranda</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Fan</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Qi</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Brannock</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>Kosten</surname>
              <given-names>TR</given-names>
            </name>
            <name name-style="western">
              <surname>Ryan</surname>
              <given-names>ND</given-names>
            </name>
            <name name-style="western">
              <surname>Kirisci</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>DeepBiomarker2: prediction of alcohol and substance use disorder risk in post-traumatic stress disorder patients using electronic medical records and multiple social determinants of health</article-title>
          <source>J Pers Med</source>
          <year>2024</year>
          <volume>14</volume>
          <issue>1</issue>
          <fpage>94</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=jpm14010094"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/jpm14010094</pub-id>
          <pub-id pub-id-type="medline">38248795</pub-id>
          <pub-id pub-id-type="pii">jpm14010094</pub-id>
          <pub-id pub-id-type="pmcid">PMC10817272</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Savova</surname>
              <given-names>GK</given-names>
            </name>
            <name name-style="western">
              <surname>Masanz</surname>
              <given-names>JJ</given-names>
            </name>
            <name name-style="western">
              <surname>Ogren</surname>
              <given-names>PV</given-names>
            </name>
            <name name-style="western">
              <surname>Zheng</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sohn</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kipper-Schuler</surname>
              <given-names>KC</given-names>
            </name>
            <name name-style="western">
              <surname>Chute</surname>
              <given-names>CG</given-names>
            </name>
          </person-group>
          <article-title>Mayo clinical text analysis and knowledge extraction system (cTAKES): architecture, component evaluation and applications</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2010</year>
          <volume>17</volume>
          <issue>5</issue>
          <fpage>507</fpage>
          <lpage>513</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/20819853"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/jamia.2009.001560</pub-id>
          <pub-id pub-id-type="medline">20819853</pub-id>
          <pub-id pub-id-type="pii">17/5/507</pub-id>
          <pub-id pub-id-type="pmcid">PMC2995668</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Devlin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chang</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Toutanova</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>BERT: pre-training of deep bidirectional transformers for language understanding</article-title>
          <source>ArXiv. Preprint posted online on October 11, 2018</source>
          <pub-id pub-id-type="doi">10.48550/arXiv.1810.04805</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cer</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Kong</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hua</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Limtiaco</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Constant</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Guajardo-Cespedes</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Yuan</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Universal Sentence Encoder</article-title>
          <source>ArXiv. Preprint posted online on March 29, 2018</source>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1803.11175"/>
          </comment>
          <pub-id pub-id-type="doi">10.48550/arXiv.1803.11175</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alsentzer</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Boag</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Weng</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Jindi</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Naumann</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>McDermott</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Publicly available clinical BERT embeddings</article-title>
          <year>2019</year>
          <conf-name>Proceedings of the 2nd Clinical Natural Language Processing Workshop</conf-name>
          <conf-date>June 7, 2019</conf-date>
          <conf-loc>Minneapolis, Minnesota</conf-loc>
          <pub-id pub-id-type="doi">10.18653/v1/w19-1909</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>HK</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>You</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Bae</surname>
              <given-names>YS</given-names>
            </name>
          </person-group>
          <article-title>Identifying alcohol-related information from unstructured bilingual clinical notes with multilingual transformers</article-title>
          <source>IEEE Access</source>
          <year>2023</year>
          <volume>11</volume>
          <fpage>16066</fpage>
          <lpage>16075</lpage>
          <pub-id pub-id-type="doi">10.1109/access.2023.3245523</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Khandve</surname>
              <given-names>SI</given-names>
            </name>
            <name name-style="western">
              <surname>Wagh</surname>
              <given-names>VK</given-names>
            </name>
            <name name-style="western">
              <surname>Wani</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>IM</given-names>
            </name>
            <name name-style="western">
              <surname>Joshi</surname>
              <given-names>RB</given-names>
            </name>
          </person-group>
          <article-title>Hierarchical neural network approaches for long document classification</article-title>
          <year>2022</year>
          <conf-name>Proceedings of the 2022 14th International Conference on Machine Learning and Computing (ICMLC ’22)</conf-name>
          <conf-date>February 18-21, 2022</conf-date>
          <conf-loc>Guangzhou China</conf-loc>
          <publisher-loc>USA</publisher-loc>
          <fpage>115</fpage>
          <lpage>119</lpage>
          <pub-id pub-id-type="doi">10.1145/3529836.3529935</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <collab>American Psychiatric Association</collab>
          </person-group>
          <source>Diagnostic and Statistical Manual of Mental Disorders. 5th Edition</source>
          <year>2013</year>
          <publisher-loc>Arlington, Virginia</publisher-loc>
          <publisher-name>American Psychiatric Publishing</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zawistowski</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Fritsche</surname>
              <given-names>LG</given-names>
            </name>
            <name name-style="western">
              <surname>Pandit</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Vanderwerff</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Patil</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Schmidt</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>VandeHaar</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Willer</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Brummett</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Kheterpal</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Zhou</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Boehnke</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Abecasis</surname>
              <given-names>GR</given-names>
            </name>
            <name name-style="western">
              <surname>Zöllner</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The Michigan Genomics Initiative: a biobank linking genotypes and electronic clinical records in Michigan Medicine patients</article-title>
          <source>Cell Genom</source>
          <year>2023</year>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>100257</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S2666-979X(23)00013-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.xgen.2023.100257</pub-id>
          <pub-id pub-id-type="medline">36819667</pub-id>
          <pub-id pub-id-type="pii">S2666-979X(23)00013-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC9932985</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Stenetorp</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Pyysalo</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Topi?</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Ohta</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Ananiadou</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Tsujii</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>brat: a web-based tool for NLP-assisted text annotation</article-title>
          <year>2012</year>
          <conf-name>Proceedings of the Demonstrations at the 13th Conference of the European Chapter of the Association for Computational Linguistics</conf-name>
          <conf-date>April 23-27, 2012</conf-date>
          <conf-loc>Avignon, France</conf-loc>
          <fpage>102</fpage>
          <lpage>107</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="web">
          <article-title>Understanding alcohol drinking patterns</article-title>
          <source>National Institute on Alcohol Abuse and Alcoholism</source>
          <year>2026</year>
          <access-date>2026-05-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.niaaa.nih.gov/alcohols-effects-health/alcohol-drinking-patterns">https://www.niaaa.nih.gov/alcohols-effects-health/alcohol-drinking-patterns</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rajapakse</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Rubenzahl</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Saripella</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>JW</given-names>
            </name>
            <name name-style="western">
              <surname>Mansouri</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Fan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Englesakis</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Preoperative alcohol use disorders and adverse outcomes in surgical patients: a systematic review and meta-analysis</article-title>
          <source>J Clin Anesth</source>
          <year>2026</year>
          <volume>111</volume>
          <fpage>112194</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0952-8180(26)00072-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jclinane.2026.112194</pub-id>
          <pub-id pub-id-type="medline">41922128</pub-id>
          <pub-id pub-id-type="pii">S0952-8180(26)00072-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Eyre</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Chapman</surname>
              <given-names>AB</given-names>
            </name>
            <name name-style="western">
              <surname>Peterson</surname>
              <given-names>KS</given-names>
            </name>
            <name name-style="western">
              <surname>Shi</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Alba</surname>
              <given-names>PR</given-names>
            </name>
            <name name-style="western">
              <surname>Jones</surname>
              <given-names>MM</given-names>
            </name>
            <name name-style="western">
              <surname>Box</surname>
              <given-names>TL</given-names>
            </name>
            <name name-style="western">
              <surname>DuVall</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Patterson</surname>
              <given-names>OV</given-names>
            </name>
          </person-group>
          <article-title>Launching into clinical space with medspaCy: a new clinical text processing toolkit in Python</article-title>
          <source>AMIA Annu Symp Proc</source>
          <year>2021</year>
          <volume>2021</volume>
          <fpage>438</fpage>
          <lpage>447</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/35308962"/>
          </comment>
          <pub-id pub-id-type="medline">35308962</pub-id>
          <pub-id pub-id-type="pii">3576697</pub-id>
          <pub-id pub-id-type="pmcid">PMC8861690</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Paszke</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gross</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Massa</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Lerer</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bradbury</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chanan</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Killeen</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Gimelshein</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Antiga</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Pytorch: an imperative style, high-performance deep learning library</article-title>
          <year>2019</year>
          <conf-name>Advances in Neural Information Processing Systems 32 (NeurIPS 2019)</conf-name>
          <conf-date>December 8-14, 2019</conf-date>
          <conf-loc>Vancouver, British Columbia</conf-loc>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://proceedings.neurips.cc/paper_files/paper/2019/hash/bdbca288fee7f92f2bfa9f7012727740-Abstract.html"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Biewald</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Experiment tracking with Weights and Biases</article-title>
          <source>Weights &amp; Biases</source>
          <year>2020</year>
          <access-date>2026-10-06</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://wandb.com">https://wandb.com</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>EB</given-names>
            </name>
          </person-group>
          <article-title>Probable inference, the law of succession, and statistical inference</article-title>
          <source>Journal of the American Statistical Association</source>
          <year>1927</year>
          <volume>22</volume>
          <issue>158</issue>
          <fpage>209</fpage>
          <lpage>212</lpage>
          <pub-id pub-id-type="doi">10.1080/01621459.1927.10502953</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Newcombe</surname>
              <given-names>RG</given-names>
            </name>
          </person-group>
          <article-title>Two-sided confidence intervals for the single proportion: comparison of seven methods</article-title>
          <source>Stat Med</source>
          <year>1998</year>
          <volume>17</volume>
          <issue>8</issue>
          <fpage>857</fpage>
          <lpage>872</lpage>
          <pub-id pub-id-type="doi">10.1002/(sici)1097-0258(19980430)17:8&lt;857::aid-sim777&gt;3.0.co;2-e</pub-id>
          <pub-id pub-id-type="medline">9595616</pub-id>
          <pub-id pub-id-type="pii">10.1002/(SICI)1097-0258(19980430)17:8&lt;857::AID-SIM777&gt;3.0.CO;2-E</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mercurio</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Garcia</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Ruest</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Duffy</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Eickhoff</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>One third of alcohol use disorder diagnoses are missed by ICD coding</article-title>
          <source>Subst Use Addctn J</source>
          <year>2025</year>
          <volume>46</volume>
          <issue>2</issue>
          <fpage>328</fpage>
          <lpage>336</lpage>
          <pub-id pub-id-type="doi">10.1177/29767342241288112</pub-id>
          <pub-id pub-id-type="medline">39511779</pub-id>
          <pub-id pub-id-type="pmcid">PMC12093275</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
