<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v26i1e51640</article-id>
      <article-id pub-id-type="pmid">38319694</article-id>
      <article-id pub-id-type="doi">10.2196/51640</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Short Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Short Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Limitations of the Cough Sound-Based COVID-19 Diagnosis Artificial Intelligence Model and its Future Direction: Longitudinal Observation Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Leung</surname>
            <given-names>Tiffany</given-names>
          </name>
        </contrib>
        <contrib contrib-type="editor">
          <name>
            <surname>de Azevedo Cardoso</surname>
            <given-names>Taiane</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Hua</surname>
            <given-names>Long</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Bossley</surname>
            <given-names>Cara</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author">
          <name name-style="western">
            <surname>Kim</surname>
            <given-names>Jina</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0009-0006-4661-7905</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Choi</surname>
            <given-names>Yong Sung</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9181-7849</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Lee</surname>
            <given-names>Young Joo</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-5294-7368</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Yeo</surname>
            <given-names>Seung Geun</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8021-1024</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Kim</surname>
            <given-names>Kyung Won</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1532-5970</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Kim</surname>
            <given-names>Min Seo</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-2115-7835</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Rahmati</surname>
            <given-names>Masoud</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff4" ref-type="aff">4</xref>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4792-027X</ext-link>
        </contrib>
        <contrib id="contrib8" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Yon</surname>
            <given-names>Dong Keon</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff6" ref-type="aff">6</xref>
          <xref rid="aff7" ref-type="aff">7</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1628-9948</ext-link>
        </contrib>
        <contrib id="contrib9" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Lee</surname>
            <given-names>Jinseok</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Department of Biomedical Engineering, Kyung Hee University</institution>
            <addr-line>1732 Deogyeong-daero, Giheung-gu</addr-line>
            <addr-line>Seoul, 17104</addr-line>
            <country>Republic of Korea</country>
            <fax>82 504 478 0201</fax>
            <phone>82 2 6935 2476</phone>
            <email>gonasago@khu.ac.kr</email>
          </address>
          <xref rid="aff8" ref-type="aff">8</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-8580-490X</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Biomedical Engineering, Kyung Hee University</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Radiology and Research Institute of Radiology, Asan Image Metrics, Clinical Trial Center, Asan Medical Center, University of Ulsan College of Medicine</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Medical and Population Genetics and Cardiovascular Disease Initiative, Broad Institute of MIT and Harvard</institution>
        <addr-line>Cambridge, MA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Department of Physical Education and Sport Sciences, Faculty of Literature and Human Sciences, Lorestan University</institution>
        <addr-line>Khoramabad</addr-line>
        <country>Iran</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>Department of Physical Education and Sport Sciences, Faculty of Literature and Humanities, Vali-E-Asr University of Rafsanjan</institution>
        <addr-line>Rafsanjan</addr-line>
        <country>Iran</country>
      </aff>
      <aff id="aff6">
        <label>6</label>
        <institution>Center for Digital Health, Medical Science Research Institute, Kyung Hee University Medical Center, Kyung Hee University College of Medicine</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff7">
        <label>7</label>
        <institution>Department of Pediatrics, Kyung Hee University Medical Center, Kyung Hee University College of Medicine</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff8">
        <label>8</label>
        <institution>Department of Electronics and Information Convergence Engineering, Kyung Hee University</institution>
        <addr-line>Yongin</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Jinseok Lee <email>gonasago@khu.ac.kr</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2024</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>6</day>
        <month>2</month>
        <year>2024</year>
      </pub-date>
      <volume>26</volume>
      <elocation-id>e51640</elocation-id>
      <history>
        <date date-type="received">
          <day>6</day>
          <month>8</month>
          <year>2023</year>
        </date>
        <date date-type="rev-request">
          <day>27</day>
          <month>10</month>
          <year>2023</year>
        </date>
        <date date-type="rev-recd">
          <day>10</day>
          <month>11</month>
          <year>2023</year>
        </date>
        <date date-type="accepted">
          <day>2</day>
          <month>1</month>
          <year>2024</year>
        </date>
      </history>
      <copyright-statement>©Jina Kim, Yong Sung Choi, Young Joo Lee, Seung Geun Yeo, Kyung Won Kim, Min Seo Kim, Masoud Rahmati, Dong Keon Yon, Jinseok Lee. Originally published in the Journal of Medical Internet Research (https://www.jmir.org), 06.02.2024.</copyright-statement>
      <copyright-year>2024</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on https://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://www.jmir.org/2024/1/e51640" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>The outbreak of SARS-CoV-2 in 2019 has necessitated the rapid and accurate detection of COVID-19 to manage patients effectively and implement public health measures. Artificial intelligence (AI) models analyzing cough sounds have emerged as promising tools for large-scale screening and early identification of potential cases.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aimed to investigate the efficacy of using cough sounds as a diagnostic tool for COVID-19, considering the unique acoustic features that differentiate positive and negative cases. We investigated whether an AI model trained on cough sound recordings from specific periods, especially the early stages of the COVID-19 pandemic, were applicable to the ongoing situation with persistent variants.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>We used cough sound recordings from 3 data sets (Cambridge, Coswara, and Virufy) representing different stages of the pandemic and variants. Our AI model was trained using the Cambridge data set with subsequent evaluation against all data sets. The performance was analyzed based on the area under the receiver operating curve (AUC) across different data measurement periods and COVID-19 variants.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>The AI model demonstrated a high AUC when tested with the Cambridge data set, indicative of its initial effectiveness. However, the performance varied significantly with other data sets, particularly in detecting later variants such as Delta and Omicron, with a marked decline in AUC observed for the latter. These results highlight the challenges in maintaining the efficacy of AI models against the backdrop of an evolving virus.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>While AI models analyzing cough sounds offer a promising noninvasive and rapid screening method for COVID-19, their effectiveness is challenged by the emergence of new virus variants. Ongoing research and adaptations in AI methodologies are crucial to address these limitations. The adaptability of AI models to evolve with the virus underscores their potential as a foundational technology for not only the current pandemic but also future outbreaks, contributing to a more agile and resilient global health infrastructure.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>COVID-19 variants</kwd>
        <kwd>cough sound</kwd>
        <kwd>artificial intelligence</kwd>
        <kwd>diagnosis</kwd>
        <kwd>human lifestyle</kwd>
        <kwd>SARS-CoV-2</kwd>
        <kwd>AI model</kwd>
        <kwd>cough</kwd>
        <kwd>sound-based</kwd>
        <kwd>diagnosis</kwd>
        <kwd>sounds app</kwd>
        <kwd>development</kwd>
        <kwd>COVID-19</kwd>
        <kwd>AI</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>In 2019, the outbreak of SARS-CoV-2 set off a global pandemic, substantially impacting human lifestyles and posing an unprecedented challenge to health care systems worldwide [<xref ref-type="bibr" rid="ref1">1</xref>]. Given its high infectivity, the urgent need for rapid and accurate COVID-19 detection has become paramount for effective patient management, timely isolation, and implementation of appropriate public health measures. Consequently, numerous studies have been undertaken to devise efficient methods for detecting infection status quickly and easily. Among these studies, artificial intelligence (AI) models that leverage prominent symptoms, such as coughing, have shown potential [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref3">3</xref>]. The ability to discern COVID-19 solely based on analyzing recorded cough sounds presents a straightforward and swift approach to screening individuals. These AI models offer hope for rapid, large-scale screening and early identification of potential cases. However, it is essential to acknowledge certain limitations in these research trends, particularly regarding variations in symptoms attributed to different SARS-CoV-2 variants [<xref ref-type="bibr" rid="ref4">4</xref>]. The emergence of new variants has introduced complexities in symptom profiles, making it more challenging to rely solely on cough sounds for accurate COVID-19 detection. As the virus continues to evolve, ongoing research and adaptations in AI models will be crucial to address these challenges and improve the accuracy and effectiveness of diagnostic approaches. Particularly, patients with the Omicron variant have shown a decreased occurrence of characteristic symptoms such as loss of taste and smell, while cold-like symptoms such as sneezing and a blocked nose have increased [<xref ref-type="bibr" rid="ref5">5</xref>]. In essence, the symptoms of COVID-19 have been increasingly resembling those of a common cold. This implies that the previous AI models designed to detect COVID-19 using cough sounds may not work as effectively or may not be as reliable both currently and in the future.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Overview</title>
        <p>In this study, we first used cough sounds as a diagnostic tool for COVID-19 stems from distinctive acoustic features that can differentiate between positive and negative cases. Building on prior research that used mel spectrograms to discern variance in cough sound spectra, our study adopts the variable frequency complex demodulation technique. Variable frequency complex demodulation offers a higher resolution than mel spectrograms, thus amplifying the discernibility of these patterns [<xref ref-type="bibr" rid="ref6">6</xref>]. As depicted in Figure S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>, the cough sound spectra of patients with COVID-19 exhibit a unique irregular spectral intensity distribution, characterized by an initial decline followed by a subsequent rise over time. In contrast, the spectra from non–COVID-19 groups display a more uniform pattern, typically showing either a consistent distribution or a gradual decline in intensity. Furthermore, our analysis extends to the differentiation in audio signal features between the 2 patient groups. The examination of 5 selected audio signal features indicates clear distinctions, which are consistent across different data sets. Illustrated in Figure S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>], we observe that the mean spectral roll-off for patients with SARS-CoV-2 positive trends toward lower values compared with that of negative individuals. Conversely, the mean spectral bandwidth is generally higher for those with COVID-19. Additional features, such as the SD of the spectral centroid, present lower values for positive cases, whereas the SD of spectral bandwidth is elevated, and the SD of the zero-crossing rate is reduced when compared with negative cases.</p>
        <p>Furthermore, we aimed to demonstrate that an AI model trained on cough sound recordings from a specific period, especially the early stages of the COVID-19 pandemic, is not applicable to the ongoing situation with persistent variants. To illustrate these findings, we used 3 cough data sets named Cambridge, Coswara, and Virufy [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. The Cambridge data set consists of audio recordings of variable-length cough audio collected through the COVID-19 Sounds App, developed by the University of Cambridge. The data measurement period encompasses the early stages of the wild type and Alpha variant occurrences (April 30, 2020, to April 26, 2021). We trained our AI model using the Cambridge data set. More specifically, we split the data into 8:2 for train and test data. Subsequently, for the train data, we performed 3-fold cross-validation and evaluated the model using the test data. The Virufy data set also includes crowdsourced cough sounds to identify patterns that signify respiratory diseases, such as COVID-19. The data measurement period primarily covers the early stages when the wild type was present (April 9, 2020, to November 26, 2020). We used the data set as additional test data. The Coswara data set also includes worldwide crowdsourced data collected through a website app, including cough, breath, and voice recordings for COVID-19 diagnosis. The data measurement period extended from the Alpha variant to the Omicron variant occurrences. We also used the data set as additional test data. To evaluate the performance of our AI model based on different variants, we categorized the data into 3 periods, each dominated by the Alpha, Delta, and Omicron variants, respectively. We summarized the data sets, along with the periods and COVID-19 variant status in which they were measured, in <xref rid="figure1" ref-type="fig">Figure 1</xref> and Tables S1 and S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>].</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Area under the receiver operating curve (AUC) values for each data set and measurement period.</p>
          </caption>
          <graphic xlink:href="jmir_v26i1e51640_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Ethical Considerations</title>
        <p>The protocol of this study was approved by the institutional review board of Kyung Hee University Hospital (KHUH 2022-06-042).</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>After the cough sound preprocessing, we developed an AI model to detect COVID-19 from extracted features and time-frequency spectrum using variable frequency complex demodulation (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>]). Table S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>] summarizes the performance of our proposed AI model according to each data set and data measurement period. The results show that the cross-validation and test results from the Cambridge data set overlap, with an area under the receiver operating curve (AUC) of 0.93. Furthermore, the Virufy data set, which was measured around a similar period as the Cambridge data set, also demonstrated a similar performance: the AUC was 0.92. On the other hand, in the case of the Coswara data set, divided into 3 groups based on the variants, it was observed that the AUC for Alpha was 0.83; for Delta, 0.77; and for Omicron, 0.55; this indicates a decline in performance as the variants progressed. <xref rid="figure1" ref-type="fig">Figures 1</xref> and <xref rid="figure2" ref-type="fig">2</xref> summarize the overall performance for each data set and measurement period.</p>
      <fig id="figure2" position="float">
        <label>Figure 2</label>
        <caption>
          <p>Our proposed model architecture. BN: batch normalization; FC: fully connected; PD: prediction; SB: spectral bandwidth; SD SC: SD of spectral centroid; SD ZCR: SD of zero crossing rate; SD SB: SD of spectral bandwidth; SR: spectral rolloff.</p>
        </caption>
        <graphic xlink:href="jmir_v26i1e51640_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>The fight against COVID-19 has been marked by the deployment of various traditional diagnostic methods, each serving as a critical pillar in our response to the pandemic. Central among these is the reverse transcription-polymerase chain reaction test, widely regarded as the gold standard for its high specificity and sensitivity [<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref13">13</xref>]. Rapid antigen tests and serological assays have also been instrumental, offering quick screening and insight into past infections, respectively. Despite their strengths, these methods have limitations, including resource dependency, time constraints, and varying degrees of accuracy. In this context, AI has emerged as a powerful aid, augmenting traditional diagnostic methods and addressing their shortcomings. By facilitating enhanced data analysis and interpretation, AI algorithms have the potential to streamline reverse transcription-polymerase chain reaction workflows, improve the reliability of antigen tests, and provide a more nuanced understanding of serological data. The integration of AI extends further to the analysis of medical imaging, where it aids in identifying patterns indicative of COVID-19, thus offering a valuable complement to molecular testing [<xref ref-type="bibr" rid="ref14">14</xref>-<xref ref-type="bibr" rid="ref16">16</xref>].</p>
      <p>Our research has focused on the novel application of AI in analyzing cough sounds, a symptom-based method with promise for noninvasive, rapid screening. We demonstrated that AI models, when trained on cough sounds, could provide a swift approach to preliminary screening, though their performance varied with the emergence of new variants. Despite the performance variability, the inherent adaptability of AI models ensures their lasting relevance. These models can be reconfigured as pretrained frameworks ready to be fine-tuned against emerging viral strains. Such adaptability allows for the rapid deployment of AI tools in response to evolving pathogens, showcasing the potential of AI to serve not only the current pandemic but also as a foundational technology for future outbreaks. The capacity of these AI models to evolve in step with the virus paves the way for an agile diagnostic ecosystem that can quickly adapt to new threats, ultimately contributing to a more resilient global health infrastructure.</p>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Supplemental tables and figures.</p>
        <media xlink:href="jmir_v26i1e51640_app1.pdf" xlink:title="PDF File  (Adobe PDF File), 589 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AI</term>
          <def>
            <p>artificial intelligence</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">AUC</term>
          <def>
            <p>area under the receiver operating curve</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This research was supported by a grant from the Korea Health Technology R&#38;D Project through the Korea Health Industry Development Institute, funded by the Ministry of Health &#38; Welfare, Republic of Korea (HV22C0233), by a National Research Foundation of Korea grant funded by the Korea government (RS-2023-00253081), and by the Hyundai Motor Chunag Mong-Koo Foundation. The authors did not use generative AI tools in writing the manuscript.</p>
    </ack>
    <notes>
      <sec>
        <title>Data Availability</title>
        <p>The data sets generated or analyzed during this study are available from the corresponding author on reasonable request.</p>
      </sec>
    </notes>
    <fn-group>
      <fn fn-type="con">
        <p>JL had full access to all of the data in the study and took responsibility for the integrity of the AI model and performance analysis. All authors approved the final version before submission. The study’s concept and design were devised by DKY and JL. The acquisition was led by JK and JL. Model development was spearheaded by JK. Analysis and interpretation involved DKY and JL. Validation was performed by YSC, YJL, SGY, and KWK. Drafting of the manuscript was done by JK, DKY, and JL. Critical revision of the manuscript for important intellectual content was a collective effort of all authors. Study supervision was undertaken by DKY and JL. JL and DKY supervised the study and are guarantors for this study. JL and DKY contributed equally as corresponding authors. The corresponding authors attest that all listed authors meet authorship criteria and that no others meeting the criteria have been omitted.</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cha</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Jung</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Seo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rahmati</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The emerging pandemic recent: SARS-CoV-2</article-title>
          <source>Life Cycle</source>
          <year>2023</year>
          <volume>3</volume>
          <fpage>e2</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.elifecycle.org/archive/view_article?pid=lc-3-0-2"/>
          </comment>
          <pub-id pub-id-type="doi">10.54724/lc.2023.e2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Laguarta</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Hueto</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Subirana</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>COVID-19 artificial intelligence diagnosis using only cough recordings</article-title>
          <source>IEEE Open J Eng Med Biol</source>
          <year>2020</year>
          <volume>1</volume>
          <fpage>275</fpage>
          <lpage>281</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/34812418"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/OJEMB.2020.3026928</pub-id>
          <pub-id pub-id-type="medline">34812418</pub-id>
          <pub-id pub-id-type="pmcid">PMC8545024</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Andreu-Perez</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Perez-Espinosa</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Timonet</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Kiani</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Giron-Perez</surname>
              <given-names>MI</given-names>
            </name>
            <name name-style="western">
              <surname>Benitez-Trinidad</surname>
              <given-names>AB</given-names>
            </name>
            <name name-style="western">
              <surname>Jarchi</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Rosales-Perez</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gatzoulis</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Reyes-Galaviz</surname>
              <given-names>OF</given-names>
            </name>
            <name name-style="western">
              <surname>Torres-Garcia</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Reyes-Garcia</surname>
              <given-names>Ca</given-names>
            </name>
            <name name-style="western">
              <surname>Ali</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Rivas</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>A generic deep learning based cough analysis system from clinically validated samples for point-of-need Covid-19 test and severity levels</article-title>
          <source>IEEE Trans Serv Comput</source>
          <year>2022</year>
          <volume>15</volume>
          <issue>3</issue>
          <fpage>1220</fpage>
          <lpage>1232</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/35936760"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/TSC.2021.3061402</pub-id>
          <pub-id pub-id-type="medline">35936760</pub-id>
          <pub-id pub-id-type="pmcid">PMC9328729</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>YJ</given-names>
            </name>
            <name name-style="western">
              <surname>Acharya</surname>
              <given-names>KP</given-names>
            </name>
          </person-group>
          <article-title>How serious is the Omicron variant? transmissibility, genomics, and responses to COVID-19 vaccines, and 'Stealth' Omicron variants</article-title>
          <source>Life Cycle</source>
          <year>2022</year>
          <volume>2</volume>
          <fpage>e7</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.elifecycle.org/archive/view_article?pid=lc-2-0-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.54724/lc.2022.e7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Whitaker</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Elliott</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bodinier</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Barclay</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Cooke</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Donnelly</surname>
              <given-names>CA</given-names>
            </name>
            <name name-style="western">
              <surname>Chadeau-Hyam</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Elliott</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Variant-specific symptoms of COVID-19 in a study of 1,542,510 adults in England</article-title>
          <source>Nat Commun</source>
          <year>2022</year>
          <volume>13</volume>
          <issue>1</issue>
          <fpage>6856</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41467-022-34244-2"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41467-022-34244-2</pub-id>
          <pub-id pub-id-type="medline">36369151</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41467-022-34244-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC9651890</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Siu</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Ju</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Chon</surname>
              <given-names>KH</given-names>
            </name>
          </person-group>
          <article-title>A high resolution approach to estimating time-frequency spectra and their amplitudes</article-title>
          <source>Ann Biomed Eng</source>
          <year>2006</year>
          <volume>34</volume>
          <issue>2</issue>
          <fpage>326</fpage>
          <lpage>338</lpage>
          <pub-id pub-id-type="doi">10.1007/s10439-005-9035-y</pub-id>
          <pub-id pub-id-type="medline">16463086</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Umapathy</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Krishnan</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Trends in audio signal feature extraction methods</article-title>
          <source>Applied Acoustics</source>
          <year>2020</year>
          <month>01</month>
          <volume>158</volume>
          <fpage>107020</fpage>
          <pub-id pub-id-type="doi">10.1016/j.apacoust.2019.107020</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chang</surname>
              <given-names>Anne B</given-names>
            </name>
          </person-group>
          <article-title>The physiology of cough</article-title>
          <source>Paediatr Respir Rev</source>
          <year>2006</year>
          <month>03</month>
          <volume>7</volume>
          <issue>1</issue>
          <fpage>2</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1016/j.prrv.2005.11.009</pub-id>
          <pub-id pub-id-type="medline">16473809</pub-id>
          <pub-id pub-id-type="pii">S1526-0542(05)00110-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chollet</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Xception: deep learning with depthwise separable convolutions</article-title>
          <year>2017</year>
          <conf-name>2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)</conf-name>
          <conf-date>2017</conf-date>
          <conf-loc>Honolulu, HI</conf-loc>
          <fpage>1800</fpage>
          <lpage>1807</lpage>
          <pub-id pub-id-type="doi">10.1109/CVPR.2017.195</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Orlandic</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Teijeiro</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Atienza</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>The COUGHVID crowdsourcing dataset, a corpus for the study of large-scale cough analysis algorithms</article-title>
          <source>Sci Data</source>
          <year>2021</year>
          <volume>8</volume>
          <issue>1</issue>
          <fpage>156</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41597-021-00937-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41597-021-00937-4</pub-id>
          <pub-id pub-id-type="medline">34162883</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41597-021-00937-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC8222356</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bhattacharya</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>NK</given-names>
            </name>
            <name name-style="western">
              <surname>Dutta</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Chetupalli</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Mote</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ganapathy</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chandrakiran</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Nori</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Suhail</surname>
              <given-names>KK</given-names>
            </name>
            <name name-style="western">
              <surname>Gonuguntla</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Alagesan</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Coswara: a respiratory sounds and symptoms dataset for remote screening of SARS-CoV-2 infection</article-title>
          <source>Sci Data</source>
          <year>2023</year>
          <volume>10</volume>
          <issue>1</issue>
          <fpage>397</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41597-023-02266-0"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41597-023-02266-0</pub-id>
          <pub-id pub-id-type="medline">37349364</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41597-023-02266-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC10287715</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Won</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>BY</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>CJ</given-names>
            </name>
          </person-group>
          <article-title>Optimization of primer sets and detection protocols for SARS-CoV-2 of coronavirus disease 2019 (COVID-19) using PCR and real-time PCR</article-title>
          <source>Exp Mol Med</source>
          <year>2020</year>
          <volume>52</volume>
          <issue>6</issue>
          <fpage>963</fpage>
          <lpage>977</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s12276-020-0452-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s12276-020-0452-7</pub-id>
          <pub-id pub-id-type="medline">32546849</pub-id>
          <pub-id pub-id-type="pii">10.1038/s12276-020-0452-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC7295692</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tombuloglu</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Sabit</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Khallaf</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kabanja</surname>
              <given-names>JH</given-names>
            </name>
            <name name-style="western">
              <surname>Alsaeed</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Saleh</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Suhaimi</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Multiplex real-time RT-PCR method for the diagnosis of SARS-CoV-2 by targeting viral N, RdRP and human RP genes</article-title>
          <source>Sci Rep</source>
          <year>2022</year>
          <volume>12</volume>
          <issue>1</issue>
          <fpage>2853</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41598-022-06977-z"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41598-022-06977-z</pub-id>
          <pub-id pub-id-type="medline">35181721</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41598-022-06977-z</pub-id>
          <pub-id pub-id-type="pmcid">PMC8857243</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Yon</surname>
              <given-names>DK</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>WH</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>TS</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>KW</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Development and validation of a deep learning model to diagnose COVID-19 using time-series heart rate values before the onset of symptoms</article-title>
          <source>J Med Virol</source>
          <year>2023</year>
          <volume>95</volume>
          <issue>2</issue>
          <fpage>e28462</fpage>
          <pub-id pub-id-type="doi">10.1002/jmv.28462</pub-id>
          <pub-id pub-id-type="medline">36602055</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kang</surname>
              <given-names>WS</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>KW</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Song</surname>
              <given-names>HO</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>TY</given-names>
            </name>
            <name name-style="western">
              <surname>Seo</surname>
              <given-names>JH</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Prediction and feature importance analysis for severity of COVID-19 in South Korea using artificial intelligence: model development and validation</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <volume>23</volume>
          <issue>4</issue>
          <fpage>e27060</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/4/e27060/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/27060</pub-id>
          <pub-id pub-id-type="medline">33764883</pub-id>
          <pub-id pub-id-type="pii">v23i4e27060</pub-id>
          <pub-id pub-id-type="pmcid">PMC8057199</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kang</surname>
              <given-names>WS</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>DW</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>CR</given-names>
            </name>
            <name name-style="western">
              <surname>Ko</surname>
              <given-names>RE</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Seo</surname>
              <given-names>JH</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>TY</given-names>
            </name>
            <name name-style="western">
              <surname>Jaimes</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>KW</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>An artificial intelligence model to predict the mortality of COVID-19 patients at hospital admission time using routine blood samples: development and validation of an ensemble model</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <volume>22</volume>
          <issue>12</issue>
          <fpage>e25442</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/12/e25442/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/25442</pub-id>
          <pub-id pub-id-type="medline">33301414</pub-id>
          <pub-id pub-id-type="pii">v22i12e25442</pub-id>
          <pub-id pub-id-type="pmcid">PMC7759509</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
