<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v22i10e19810</article-id>
      <article-id pub-id-type="pmid">33095174</article-id>
      <article-id pub-id-type="doi">10.2196/19810</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Clinical Context–Aware Biomedical Text Summarization Using Deep Neural Network: Model Development and Validation</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>ALi</surname>
            <given-names>Farman</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Chen</surname>
            <given-names>Sonia</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Afzal</surname>
            <given-names>Muhammad</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7851-2327</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Alam</surname>
            <given-names>Fakhare</given-names>
          </name>
          <degrees>MS</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9495-7051</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Malik</surname>
            <given-names>Khalid Mahmood</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <address>
            <institution>Department of Computer Science &#38; Engineering</institution>
            <institution>School of Engineering and Computer Science</institution>
            <institution>Oakland University</institution>
            <addr-line>Engineering Center 501</addr-line>
            <addr-line>115 Library Drive</addr-line>
            <addr-line>Rochester, MI, 48309</addr-line>
            <country>United States</country>
            <phone>1 2483703542</phone>
            <email>mahmood@oakland.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7927-3436</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Malik</surname>
            <given-names>Ghaus M</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1877-7024</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Software</institution>
        <institution>Sejong University</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Computer Science &#38; Engineering</institution>
        <institution>School of Engineering and Computer Science</institution>
        <institution>Oakland University</institution>
        <addr-line>Rochester, MI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Neurosurgery</institution>
        <institution>Henry Ford Hospital</institution>
        <addr-line>Detroit, MI</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Khalid Mahmood Malik <email>mahmood@oakland.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>10</month>
        <year>2020</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>23</day>
        <month>10</month>
        <year>2020</year>
      </pub-date>
      <volume>22</volume>
      <issue>10</issue>
      <elocation-id>e19810</elocation-id>
      <history>
        <date date-type="received">
          <day>2</day>
          <month>5</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>27</day>
          <month>7</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>24</day>
          <month>9</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©Muhammad Afzal, Fakhare Alam, Khalid Mahmood Malik, Ghaus M Malik. Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 23.10.2020.</copyright-statement>
      <copyright-year>2020</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on http://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://www.jmir.org/2020/10/e19810/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Automatic text summarization (ATS) enables users to retrieve meaningful evidence from big data of biomedical repositories to make complex clinical decisions. Deep neural and recurrent networks outperform traditional machine-learning techniques in areas of natural language processing and computer vision; however, they are yet to be explored in the ATS domain, particularly for medical text summarization.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>Traditional approaches in ATS for biomedical text suffer from fundamental issues such as an inability to capture clinical context, quality of evidence, and purpose-driven selection of passages for the summary. We aimed to circumvent these limitations through achieving precise, succinct, and coherent information extraction from credible published biomedical resources, and to construct a simplified summary containing the most informative content that can offer a review particular to clinical needs.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>In our proposed approach, we introduce a novel framework, termed Biomed-Summarizer, that provides quality-aware Patient/Problem, Intervention, Comparison, and Outcome (PICO)-based intelligent and context-enabled summarization of biomedical text. Biomed-Summarizer integrates the prognosis quality recognition model with a clinical context–aware model to locate text sequences in the body of a biomedical article for use in the final summary. First, we developed a deep neural network binary classifier for quality recognition to acquire scientifically sound studies and filter out others. Second, we developed a bidirectional long-short term memory recurrent neural network as a clinical context–aware classifier, which was trained on semantically enriched features generated using a word-embedding tokenizer for identification of meaningful sentences representing PICO text sequences. Third, we calculated the similarity between query and PICO text sequences using Jaccard similarity with semantic enrichments, where the semantic enrichments are obtained using medical ontologies. Last, we generated a representative summary from the high-scoring PICO sequences aggregated by study type, publication credibility, and freshness score.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Evaluation of the prognosis quality recognition model using a large dataset of biomedical literature related to intracranial aneurysm showed an accuracy of 95.41% (2562/2686) in terms of recognizing quality articles. The clinical context–aware multiclass classifier outperformed the traditional machine-learning algorithms, including support vector machine, gradient boosted tree, linear regression, K-nearest neighbor, and naïve Bayes, by achieving 93% (16127/17341) accuracy for classifying five categories: aim, population, intervention, results, and outcome. The semantic similarity algorithm achieved a significant Pearson correlation coefficient of 0.61 (0-1 scale) on a well-known BIOSSES dataset (with 100 pair sentences) after semantic enrichment, representing an improvement of 8.9% over baseline Jaccard similarity. Finally, we found a highly positive correlation among the evaluations performed by three domain experts concerning different metrics, suggesting that the automated summarization is satisfactory.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>By employing the proposed method Biomed-Summarizer, high accuracy in ATS was achieved, enabling seamless curation of research evidence from the biomedical literature to use for clinical decision-making.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>biomedical informatics</kwd>
        <kwd>automatic text summarization</kwd>
        <kwd>deep neural network</kwd>
        <kwd>word embedding</kwd>
        <kwd>semantic similarity</kwd>
        <kwd>brain aneurysm</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>Automatic text summarization (ATS) is a leading topic in the field of information retrieval research, particularly in the medical and biomedical domains, which offers an efficient solution to access the ever-growing amount of scientific and clinical literature by summarizing the source documents while maintaining their most informative contents [<xref ref-type="bibr" rid="ref1">1</xref>]. The large quantity of biomedical data now accessible to clinicians makes it challenging to locate the correct data rapidly; thus, automatic summarization can provide highlights particular to clinical needs [<xref ref-type="bibr" rid="ref2">2</xref>]. Moreover, achieving precise, succinct, and coherent information extraction from credible published biomedical resources to construct a simplified summarization plays an imperative role in clinical decision-making, educating patients, and medical education. At the same time, automating this process will provide better opportunities for users to obtain the most critical points of required clinical knowledge without having to delve into an enormous amount of text, saving hours of searching. It is highly desirable to accurately identify scientifically sound published studies and summarize selected studies against a given type (eg, intervention and prognosis) of clinical query.</p>
        <p>The primary requirements in carrying out this task of extracting noteworthy information are the efficiency of the process, contextualization, and precision of retrieved contents. Efficiency pertains to reducing human involvement and achieving the required materials in a timely manner; contextualization refers to the user objective relevant to a clinical task; and precision of contents applies to the correct identification of needed contents from trustworthy resources. Owing to the massive increase in the biomedical research literature, finding relevant, scientifically sound, and fit-to-context information has become more and more challenging for precise automatic text classification and summarization. The field of text classification and ATS is well-explored [<xref ref-type="bibr" rid="ref3">3</xref>-<xref ref-type="bibr" rid="ref7">7</xref>]; however, minimal work has been done in a clinically fit-to-context summary of biomedical text [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>]. Medical text summarization poses a unique set of challenges compared to summarization in other domains [<xref ref-type="bibr" rid="ref10">10</xref>].</p>
        <p>The main limitations of existing medical text summarization are as follows: (a) inability to capture the clinical context while ranking the sentences for the summary; (b) lack of consideration in checking the quality of the documents before performing summarization; (c) inability of identifying context and implicit information present in the biomedical text, which cannot explicitly match with the user query; (d) lack of purpose-driven ranking and selection of passage for the final summary; and (e) nonuse of a large training set for training deep neural network models. To overcome these limitations, this paper introduces a novel framework called Biomed-Summarizer, which provides a quality-aware Patient/Problem, Intervention, Comparison, and Outcome (PICO)-based intelligent and context-enabled summarization of contents of biomedical literature to satisfy the requirements listed above. Unlike the use of traditional features using a bag of words (BoW) approach, in which the text is tokenized into multiple words and each word is given a number representing the frequency of its use [<xref ref-type="bibr" rid="ref3">3</xref>], more powerful word-embedding techniques have recently been introduced, such as word2vec [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>] and GloVe [<xref ref-type="bibr" rid="ref13">13</xref>], to combat the issues of data sparseness and context-overlooking by generating a vector space, typically of several hundred dimensions [<xref ref-type="bibr" rid="ref14">14</xref>]. The word2vec approach creates vectors at the sentence or document level by employing the sentence2vec or doc2vec model [<xref ref-type="bibr" rid="ref15">15</xref>]. The Biomed-Summarizer framework uses the Keras tokenizer [<xref ref-type="bibr" rid="ref16">16</xref>] to generate a contextually rich set of features with a compact number of dimensions to enable semantics-based matching for precise extraction, summarization of contents, and avoiding sparseness. These features are then used as input to our bidirectional long-short term memory (Bi-LSTM) [<xref ref-type="bibr" rid="ref17">17</xref>] classification model to identify the PICO sequence. PICO was initially proposed for formatting a clinical question. Subsequently, researchers have used the PICO structure for information retrieval and sentence identification in the text of biomedical documents [<xref ref-type="bibr" rid="ref18">18</xref>-<xref ref-type="bibr" rid="ref21">21</xref>]. In deciding which sequences to include in the final summary, we considered a comprehensive criterion that provides information on the quality of the study to which that sequence belongs, the relevance of the sequence to the user query, study type, credibility of the publishing venue, and freshness in term of the date of publication.</p>
        <p>In summary, the main contributions of this paper are as follows. First, we introduce a novel framework, termed Biomed-Summarizer, for extractive multidocument ATS of biomedical documents. The summary construction is based on PICO elements identified in each record. Additionally, we employ contextual and quality parameters for the selection of a subset of a PICO sequence of sentences to include in the final summary.</p>
        <p>Second, for quality recognition, a highly optimized multilayer feed-forward neural network model, multilayer perceptron (MLP), is presented to acquire significantly accurate results. This model offers binary classification to identify the soundness of a study based on data (title and abstract) and metadata (article type, authors, and publishing venue and date) features.</p>
        <p>Third, for PICO elements classification, we propose a Bi-LSTM recurrent neural network (RNN) model trained on the vector representation of the text, which significantly boosts the performance compared to conventional machine-learning models such as support vector machine (SVM), logistic regression, decision tree, and naïve Bayes. Unlike previous studies that focused on the detection of PICO elements one-by-one by employing a separate binary classifier for each PICO element, the proposed approach is a multiclassifier model, which classifies PICO sequences simultaneously from any given biomedical study.</p>
        <p>Fourth, to accurately extract the PICO sentences to be included in the final summary, we present a novel method of calculating the similarity between the query and medical text using the Jaccard coefficient after semantically enriching the text using medical ontologies.</p>
        <p>Finally, we offer a publicly available dataset [<xref ref-type="bibr" rid="ref22">22</xref>] comprising thousands of abstracts related to intracranial aneurysm (also known as cerebral or brain aneurysm) curated from the biomedical literature for PICO-based classification. Additionally, another open-source dataset [<xref ref-type="bibr" rid="ref22">22</xref>] is presented for the quality recognition model.</p>
        <p>We aimed to achieve these contributions through the precise, succinct, and coherent information extraction from credible published biomedical resources, and to construct a simplified summary containing the most informative contents that provide a review particular to clinical needs.</p>
      </sec>
      <sec>
        <title>Related Works</title>
        <sec>
          <title>ATS in the Biomedical Domain</title>
          <p>Summarization techniques are generally divided into two categories: abstractive and extractive [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref4">4</xref>]. Abstractive summarization methods examine the text and generate a new summarized text as a representative of the original text. In contrast, extractive summarization methods recognize the important part of the text, extract that component, and generate the summary verbatim. Extractive summarization approaches are classified into different categories such as statistical-, topic-, graph-, discourse-, and machine learning–based approaches [<xref ref-type="bibr" rid="ref5">5</xref>]. Single and multidocument summarization are the two principal categories concerning the number of documents, whereas generic and query-specific are the two main types of summaries [<xref ref-type="bibr" rid="ref5">5</xref>]; however, another possible criterion can be used to classify these studies into item set–based mining, and classification and ranking. The item set–based mining approach extracts domain concepts in the representative sentences to generate a graph-based summary [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. The classification and ranking–based approach first detects key sentences and ranks them according to their importance in the text to produce a plain textual summary. For such a task, some researchers have used statistical features such as term frequency, sentence position, and similarity with the title [<xref ref-type="bibr" rid="ref25">25</xref>,<xref ref-type="bibr" rid="ref26">26</xref>], whereas other methods incorporated semantic information extracted from external linguistic resources [<xref ref-type="bibr" rid="ref26">26</xref>,<xref ref-type="bibr" rid="ref27">27</xref>].</p>
        </sec>
        <sec>
          <title>Identification of PICO Elements in Text</title>
          <p>Minimal work has been done in the area of PICO-based retrieval of contents from the biomedical literature. The existing studies are categorized based on three aspects: individual PICO element identification [<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref28">28</xref>], sentence classification [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref29">29</xref>], and question and answer with summarization [<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref30">30</xref>]. In a proceeding, the authors presented a process of a PICO corpus at the individual element level and sentence level [<xref ref-type="bibr" rid="ref31">31</xref>]. A hybrid approach of combining machine learning and rule-based methods was proposed for the identification of PICO sentences and individual elements in successive order [<xref ref-type="bibr" rid="ref20">20</xref>]. Another study on PICO sentence extraction was carried out with a supervised distance supervision approach that capitalizes on a small labeled dataset to mitigate noise in distantly derived annotations [<xref ref-type="bibr" rid="ref32">32</xref>]. The authors developed a naïve Bayes–based classifier and reported that it is not sufficient to rely only on the first sentence of each section, particularly when high recall is required [<xref ref-type="bibr" rid="ref28">28</xref>]. Boudin et al [<xref ref-type="bibr" rid="ref33">33</xref>] used multiple supervised classification algorithms to detect PICO elements at the sentence level by training data on structured medical abstracts for each PICO element. The results showed that the detection accuracy was better for the Patient/Problem compared to the Intervention and Outcome elements.</p>
        </sec>
        <sec>
          <title>Quality of Biomedical Studies</title>
          <p>Several promising approaches have been explored [<xref ref-type="bibr" rid="ref34">34</xref>-<xref ref-type="bibr" rid="ref41">41</xref>] to retrieve high-quality (ie, scientifically sound) studies from the PubMed database. Among these, some methods such as PubMed Clinical Queries rely on Boolean-based strategies using Medical Subject Heading (MeSH) terms and keywords [<xref ref-type="bibr" rid="ref39">39</xref>,<xref ref-type="bibr" rid="ref40">40</xref>]. Clinical Queries are often considered as a reference point for assessing the effectiveness of approaches intended to retrieve scientifically sound studies from PubMed. Some methods have also used supervised machine-learning algorithms such as SVM, naïve Bayes, and decision tree to identify scientifically rigorous studies from PubMed [<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref37">37</xref>,<xref ref-type="bibr" rid="ref38">38</xref>,<xref ref-type="bibr" rid="ref42">42</xref>]. These approaches mainly depend on bibliometric features, semantic features, and MeSH terms. The limitation of approaches that use MeSH terms is the availability, as MeSH terms are added to PubMed citations with an average time gap of 20 to 252 days after an article is published [<xref ref-type="bibr" rid="ref40">40</xref>,<xref ref-type="bibr" rid="ref43">43</xref>]. Very recently, a deep-learning approach was used for the task of detecting quality articles in PubMed [<xref ref-type="bibr" rid="ref40">40</xref>,<xref ref-type="bibr" rid="ref41">41</xref>]. Deep learning–based approaches have proven to improve accuracy over the existing approaches of machine-learning models, PubMed Clinical Queries search, and McMaster text word search in terms of precision. These approaches were trained on a treatment-related dataset.</p>
        </sec>
        <sec>
          <title>Sentence Scoring and Ranking for Summarization</title>
          <p>Various methods have been proposed to choose what text should be included in the final summary [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref44">44</xref>]. The most common method is the frequency-based approach, in which a sentence with a word holding the highest frequency is given more importance [<xref ref-type="bibr" rid="ref45">45</xref>]. Some studies have found similarities in a sentence with the title of the document [<xref ref-type="bibr" rid="ref5">5</xref>]. If a sentence encompasses words in the title, the sentence is assigned a score value of 1; otherwise, a score value of 0 is assigned. Another approach commonly followed for the selection of sentences to include in the summary is the cue words technique, in which the cue words are provided by the user [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref44">44</xref>]. A score value of 1 is assigned to the sentence if it contains these cue words; otherwise, a score value of 0 is assigned. Sentence position and length are also considered for sentence inclusion in the summary. These techniques are used to score a sentence by linearly combining the individual scores of each method [<xref ref-type="bibr" rid="ref44">44</xref>]. Based on the scoring, a certain number of sentences with high ranks are picked up to include in the final summary. Recently, deep-learning approaches have been widely used in text summarization [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref46">46</xref>-<xref ref-type="bibr" rid="ref49">49</xref>]. A neural network–based general framework was proposed for single-document summarization using a hierarchical document encoder and attention-based extractor [<xref ref-type="bibr" rid="ref48">48</xref>]. For the sentence ranking task of multidocument summarization, a ranking framework was presented using a recursive neural network (R2N2) [<xref ref-type="bibr" rid="ref49">49</xref>]. A query-focus summarization system called AttSum [<xref ref-type="bibr" rid="ref46">46</xref>] was proposed that determines ranking based on <italic>both</italic> saliency and relevance in contrast to previous approaches of learning to rank based on <italic>either</italic> relevance or saliency.</p>
          <p>The prominent issues with existing approaches are as follows: (a) inability to capture the clinical context while ranking the sentences for creating summarization, (b) omitting the check on the quality of the documents used in the summary, (c) inability of identifying implicit information located in the body text of a study that is not explicitly matched with the user query, (d) lack of purpose-driven ranking and selection of passage for the final summary, and (e) unavailability of a large training set for training deep neural network models. Biomed-Summarizer circumvents all of these limitations.</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Overall Design</title>
        <p><xref rid="figure1" ref-type="fig">Figure 1</xref> shows the architecture of the proposed Biomed-Summarizer framework highlighting four main modules: data preprocessing, quality recognition, contextual text classification, and text summarization. The main tasks in data preprocessing are sequence marker, cleaning, tokenization, vector generation, and feature creation. The quality recognition module identifies the scientific soundness of a study using a feed-forward deep-learning binary classifier that is trained on biomedical data and metadata features. The contextual text classification classifies the PICO sequence of sentences in the selected quality documents using the Bi-LSTM model, an RNN trained on a large set of data acquired from BioNLP [<xref ref-type="bibr" rid="ref21">21</xref>] and domain-specific Medline abstracts. Lastly, the summarization module generates a summary from the classified PICO sequences of multiple documents using a sequence scoring method consisting of semantic similarity (between a query and a text sequence extracted from a biomedical study), the publishing venue’s credibility, and year of publication.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Proposed Biomed-Summarizer architecture with four major components: data preprocessing, quality recognition, context identification, and summary construction. PQR: prognosis quality recognition; CCA: clinical context-aware; PICO: Population/Problem, Intervention, Comparison, Outcome.</p>
          </caption>
          <graphic xlink:href="jmir_v22i10e19810_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Data Preprocessing</title>
        <p>Biomed-Summarizer acquires data from two input sources: BioNLP dataset and PubMed. The BioNLP dataset is publicly available, whereas we retrieved the domain-specific abstracts from PubMed. The process used to parse raw abstracts retrieved from the PubMed and BioNLP is described below.</p>
        <sec>
          <title>Sequence Marker</title>
          <p>The task of sequence marker is to parse each raw abstract in the dataset to retrieve the headings based on the keywords listed in the dictionary, as shown in <xref ref-type="boxed-text" rid="box1">Textbox 1</xref>. If a keyword is matched in the abstract, we extract the text (sequence of sentences) under that heading and mark it with the corresponding label. For instance, if a heading “objective” is found in an abstract, label “A” is assigned to the text. The dictionary is based on a previous study [<xref ref-type="bibr" rid="ref21">21</xref>] with extension of a few more keywords (eg, Patient 1 and Patient 2). This process is repeated for all of the abstracts in the dataset of documents retrieved from PubMed.</p>
          <boxed-text id="box1" position="float">
            <title>Master dictionary representing keywords that appear in headings in the structured abstracts of biomedical studies.</title>
            <disp-quote>
              <p>dict = {‘A’: ['objective,' 'background,' 'background and objectives,' 'context,' 'background and purpose,'</p>
            </disp-quote>
            <disp-quote>
              <p>'purpose,' 'importance,' 'introduction,' 'aim,' 'rationale,' 'goal,' 'context,' 'hypothesis'], 'P': ['population,' 'participant,' 'sample,' 'subject,' 'patient,' 'patient 1,''patient 2'],  'I': ['intervention,' 'diagnosis'], 'O': ['outcome,' 'measure,' 'variable,' 'assessment'], M': ['method,' 'setting,' 'design,' 'material,' 'procedure,' 'process,' 'methodology'], 'R': ['result,' 'finding'], 'C': ['conclusion,' 'implication,' 'discussion,' 'interpretation table']}</p>
            </disp-quote>
          </boxed-text>
          <p>We employed the following steps to prepare data for the clinical context–aware (CCA) classification model.</p>
        </sec>
        <sec>
          <title>Text Cleaning</title>
          <p>The process of text cleaning removes special characters, punctuation, stop words, and URLs present in the text. For removing stop words, we used the Python NLKT library [<xref ref-type="bibr" rid="ref50">50</xref>].</p>
        </sec>
        <sec>
          <title>Tokenization</title>
          <p>This process splits the sentences retrieved after the text cleaning process into individual tokens. We used the Keras tokenizer [<xref ref-type="bibr" rid="ref16">16</xref>] to create a list of tokens for each biomedical paragraph.</p>
        </sec>
        <sec>
          <title>Vector Generation</title>
          <p>We considered a maximum of 50,000 words for each paragraph and generated a 250-dimension vector representation using Keras text [<xref ref-type="bibr" rid="ref16">16</xref>] to sequence functionality. We made sure that each vector length is the same.</p>
          <p>As a result of the above steps, we obtained a vector representation of each paragraph, which is then used for training and testing of the CCA classification model. An example of text sequences and corresponding class labels is shown in <xref ref-type="table" rid="table1">Table 1</xref>. It is important to mention that we do not require a process of sequence marking on BioNLP data because these sentences are premarked.</p>
          <table-wrap position="float" id="table1">
            <label>Table 1</label>
            <caption>
              <p>Example sequence of sentences for an assigned category of Aim, Population, Methods, Interventions, Results, Conclusion, and Outcomes.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="850"/>
              <col width="150"/>
              <thead>
                <tr valign="top">
                  <td>Sequence</td>
                  <td>Category</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>The aim of the present study was to evaluate whether the Anterior communicating artery (A com) aneurysms behave differently from the aneurysms located elsewhere with respect to size being a rupture risk. To this end, we examined the clinical data of ruptured A com aneurysms and analyzed other morphological parameters, including size parameter, providing adequate data for predicting rupture risk of the A com aneurysms.</td>
                  <td>Aim (A)</td>
                </tr>
                <tr valign="top">
                  <td>Between January 2010 and December 2015, a total of 130 consecutive patients at our institution with the A com aneurysms-86 ruptured and 44 unruptured-were included in this study. The ruptured group included 43 females (50%) and 43 males (50%) with the mean age of 56 years (range, 34-83 years). The unruptured group included 23 females (52%) and 21 males (48%) with the mean age of 62 years (range, 28-80 years). All patients underwent either digital subtraction angiography or 3-dimensional computed tomography angiography. The exclusion criteria for this study were the patients with fusiform, traumatic, or mycotic aneurysm. There were preexisting known risk factors, such as hypertension in 73 patients, who required antihypertensive medication; other risk factors included diabetes mellitus (9 patients), coronary heart disease (9 patients), previous cerebral stroke (18 patients), and end-stage renal disease (3 patients) in the ruptured group. In the unruptured group, 38 patients had hypertension, 4 had diabetes mellitus, 5 had coronary heart disease, 10 had a previous cerebral stroke, and 2 had end-stage renal disease.</td>
                  <td>Population (P)</td>
                </tr>
                <tr valign="top">
                  <td>Four intracranial aneurysms cases were selected for this study. Using CT angiography images, the rapid prototyping process was completed using a polyjet technology machine. The size and morphology of the prototypes were compared to brain digital subtraction arteriography of the same patients.</td>
                  <td>Methods (M)</td>
                </tr>
                <tr valign="top">
                  <td>After patients underwent dural puncture in the sitting position at L3-L4or L4-L5, 0.5% hyperbaric bupivacaine was injected over two minutes: group S7.5 received 1.5 mL, Group S5 received 1.0 mL, and group S4 received 0.8 mL. interventions after sitting for 10 minutes, patients were positioned for surgery.</td>
                  <td>Intervention (I)</td>
                </tr>
                <tr valign="top">
                  <td>The ruptured group consisted of 9 very small (&#60;2 mm), 38 small (2-4 mm), 32 medium (4-10 mm), and 7 large (&#62;10 mm) aneurysms; the unruptured group consisted of 2 very small, 16 small, 25 medium, and one large aneurysms. There were 73 ruptured aneurysms with small necks and 13 with wide necks (neck size&#62;4 mm), and 34 unruptured aneurysms with small necks and 10 with wide necks.</td>
                  <td>Results (R)</td>
                </tr>
                <tr valign="top">
                  <td>The method which we develop here could become surgical planning for intracranial aneurysm treatment in the clinical workflow.</td>
                  <td>Conclusion (C)</td>
                </tr>
                <tr valign="top">
                  <td>The prevailing view is that larger aneurysms have a greater risk of rupture. Predicting the risk of aneurysmal rupture, especially for aneurysms with a relatively small diameter, continues to be a topic of discourse. In fact, the majority of previous large-scale studies have used the maximum size of aneurysms as a predictor of aneurysm rupture.</td>
                  <td>Outcome (O)</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
        </sec>
      </sec>
      <sec>
        <title>Quality Recognition Model</title>
        <p>The proposed model, as shown in <xref rid="figure2" ref-type="fig">Figure 2</xref>, comprises multilayer feed-forwarded neural networks as a so-called MLP. The MLP is further optimized with an ensemble method using adaptive boosting (AdaBoost). The final AdaBoost-MLP, termed the prognosis quality recognition (PQR) model, was trained on a dataset acquired automatically through PubMed searches based on the following two criteria: selecting the “Clinical Query prognosis” filter and choosing scope as “narrow.” To build and evaluate the model, we performed the steps involved in the preparation of a dataset for training the deep-learning models, followed by training and tuning the deep-learning models. Additionally, we compared the performance of the proposed model with those of shallow machine-learning models in terms of precision and recall.</p>
        <p>For training and testing the PQR model, we collected a dataset consisting of a total of 2686 Medline abstracts, 697 of which were considered to be positive studies (ie, scientifically sound). To retrieve positive studies, we used the interface of Clinical Queries [<xref ref-type="bibr" rid="ref33">33</xref>]. To retrieve negative studies (ie, not scientifically sound), we used the interface of PubMed and retrieved studies that were not included in the positive set.</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Process steps of proposed prognosis quality recognition (PQR) model training and testing.</p>
          </caption>
          <graphic xlink:href="jmir_v22i10e19810_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Training and Tuning the Deep-Learning Model</title>
        <p>The PQR model is trained using five textual features, including two data features (title and abstract) and three metadata features (article type, publishing journal, and authors). The data features were preprocessed through applying the steps described above. The model consists of 5 layers with 3 hidden layers of size 100, 50, and 25, respectively. The input layers take the BoW vectors generated from data features. The “Maxout” activation function was used with 10 epochs. The data were split into a 70:30 ratio for training and testing.</p>
      </sec>
      <sec>
        <title>Comparison with Shallow Machine-Learning Models</title>
        <p>To identify the best performer on our dataset, we compared a set of machine-learning algorithms, including SVM, naïve Bayes, k-nearest neighbor, and decision tree. These algorithms were tested in other studies [<xref ref-type="bibr" rid="ref35">35</xref>,<xref ref-type="bibr" rid="ref39">39</xref>] for addressing similar problems.</p>
      </sec>
      <sec>
        <title>CCA Classification Model</title>
        <p>The CCA classification model aims to develop a multiclass classifier that can accurately classify text from selected quality documents (given by the quality model) into one of the following 7 classes: Aim, Methods, Population, Intervention, Results, Outcome, and Conclusion. We then merged Methods and Population into a single category (P) and Outcome and Conclusion into a single category (O) because of their strong correlation found in the text. Although our focus was to utilize PICO classes in the summarization task, we retained the other categories to enable additional non-PICO clinical applications such as summarization for medical education and deriving rules from clinical trials for developing the knowledge base of a clinical decision support system.</p>
        <p>PICO is a well-known terminology in evidence-based medicine. Our proposed CCA classifier incorporates the Patient/Problem, Intervention, and Outcome PICO components in addition to two more classes, Aim and Results. PICO detection is a sequential sentence classification task rather than a single sentence problem; therefore, the sequence of sentences is jointly predicted. In this way, more complete context is captured from multiple sentences, which then improves the classification accuracy of predicting the current sentence. The steps followed to build and evaluate the model were: (a) preparation of a dataset for training the deep-learning model, (b) training and tuning the deep-learning model, and (c) comparison with traditional machine-learning models in terms of precision and recall.</p>
      </sec>
      <sec>
        <title>Model Building</title>
        <p>The neural network model is heavily used in text processing due to the ability to process arbitrary-length sequences. RNNs are immensely popular in multiclass text classification. To build and evaluate the classification model, we employed the Bi-LSTM model as a type of RNN. This model preserves the long-term dependency of the text and is one of the most popular models in text classification.</p>
        <p>As shown in <xref rid="figure3" ref-type="fig">Figure 3</xref>, the input layer consists of 250 dimensions showing the numeric features created using the Keras tokenizer, embedding layer, LSTM logical hidden layers, classification layer, and output layer. In brief, the trained CCA model has the following features: (1) an initial training dataset comprising 173,401 records, 90% (n=156,060) of which were used for training and 10% (n=17,341) of which were used for testing; (2) the first hidden layer is the embedding layer that uses 100-dimension vectors to represent each paragraph; (3) the next layer is the LSTM layer with 100 memory units, and the recurrent dropout rate was set to 0.2; (4) the next layer is the dense layer with 5 nodes, and we used the SoftMax activation function for multiclassification, and categorical_crossentropy as a loss function; (5) the training dataset is further divided into two parts, 10% of which was used to validate the category and minimize the loss function, resulting in 140,454 records used for training and 15,606 used for validation; (6) the model was then trained across 5 epochs with a minibatch size of 64.</p>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>Clinical context–aware (CCA) classifier trained on 250-dimension feature vectors, 100 nodes at the embedding layer, 100 memory units of the long short-term memory (LSTM) layer logical hidden layers, and 5 classification nodes.</p>
          </caption>
          <graphic xlink:href="jmir_v22i10e19810_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>ATS</title>
        <p>For the ATS task, we developed a sentence-scoring mechanism based on a multifeatured matrix, consisting of 4 features (relevance, study type, venue credibility, and freshness), where each feature was learned with a specific method as explained in the following sections.</p>
        <p>First, an individual score value is assigned to each feature, which is then aggregated in the final column as a final score using the formula in Equation 1:</p>
        <disp-formula>Aggregate Score = <italic>βR<sub>score</sub></italic> + <italic>γ</italic>(<italic>ST<sub>rank</sub></italic> + <italic>VC<sub>score</sub></italic> + <italic>f<sub>score</sub></italic>) <bold>(1)</bold>,</disp-formula>
        <p>where <italic>R<sub>score</sub></italic> is the relevance score, <italic>ST<sub>rank</sub></italic> is the study type rank, <italic>VC<sub>score</sub></italic> is the publishing venue credibility score, <italic>f<sub>score</sub></italic> is the freshness score, and <italic>β</italic>=70 and <italic>γ</italic>=10 are the scaling constants to keep the aggregate score in the range of 0 to 100.</p>
      </sec>
      <sec>
        <title>Relevance Score</title>
        <p>For the relevance score, we developed a semantic similarity algorithm, Jaccard similarity with semantic enrichments (JS<sup>2</sup>E) as a 6-step algorithm that uses the BioPortal application programming interface to access biomedical ontologies (SNOMED CT, MedDRA, NBO, NIFSTD) for obtaining semantic enrichment and the Jaccard similarity metric to find similarities between two texts.</p>
        <p>In step 1, an individual sequence of text is obtained from the query, which is preprocessed to remove the special characters, separate characters, and different formatting tags. In step 2, the preprocessed text is annotated using BioPortal. In step 3, each token is enriched using “definition,” “synonyms,” and “prefLabel” relations from selected ontologies. In step 4, the annotations of text are retrieved using the “definition” relationship, along with the preprocessing (step 1) and annotation (step 2) procedures. In step 5, the annotated tokens received in step 2 are combined with the “synonyms” and “preflabel” obtained in step 3 and the annotated tokens received in step 4, and the data structure of metatokens is constructed. Finally, in step 6, the Jaccard similarity between the metatokens of the text sequence and the query is calculated using Equation 2:</p>
        <disp-formula><italic>R<sub>score</sub></italic>=<italic>S<sub>m</sub></italic> ∩ <italic>Q<sub>m</sub></italic>/<italic>S<sub>m</sub></italic>∪ <italic>Q<sub>m</sub></italic> <bold>(2)</bold>,</disp-formula>
        <p>where <italic>S<sub>m</sub></italic> are the metaset tokens of the text sequence in a document and <italic>Q<sub>m</sub></italic> are the metaset tokens of the query.</p>
      </sec>
      <sec>
        <title>Study Type</title>
        <p>The study type plays a vital role in proving a study’s usefulness for a user concerning a clinical task. For instance, if a surgeon wants to look for advances in successful surgical treatments, randomized control trials or a meta-analysis of randomized controlled trials will be the priority. The priority will change if the user is interested in prognosis-related studies. Study types can be used to grade evidence concerning quality and other elements [<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref51">51</xref>,<xref ref-type="bibr" rid="ref52">52</xref>]. To find the priority of different study types in the category of prognosis, we conducted a questionnaire-based survey among domain experts who were asked to assign a score value for each study type. An example of a filled-in questionnaire by a physician is provided in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
        <p>The final rank value is then learned from the average value of rank values assigned by the domain experts, as shown in Equation 3.</p>
        <disp-formula>
          <graphic xlink:href="jmir_v22i10e19810_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </disp-formula>
        <p>where <italic>V<sub>i</sub></italic> represents the rank values assigned to a study type by each domain expert, <italic>n</italic> is the total number of domain experts that participated in the questionnaire, and α=0.1 is a scaling constant to keep the final score value in the range of 0 to 1.</p>
      </sec>
      <sec>
        <title>Venue Credibility</title>
        <p>The credibility of a study publishing venue, including journals, proceedings, and books, is also an important parameter; however, it is more of a subjective matter. Therefore, it is necessary to consult with the stakeholders of the service. For this study, we sought to obtain a list of valuable publishing venues. We involved resident doctors to rank various venues concerning their credibility. An example of a filled questionnaire by a physician is provided in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. The final credibility score was determined from the average value of rank values assigned by the domain experts, as shown in Equation 4:</p>
        <disp-formula>
          <graphic xlink:href="jmir_v22i10e19810_fig6.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </disp-formula>
        <p>where <italic>S<sub>i</sub></italic> is the initial rank assigned by the domain experts, <italic>S<sub>f</sub></italic> is the mapped score obtained through mappings {1→10, 2→9, 3→8, 4→7, 5→6, and 6→5}, and α=0.1 is a scaling constant to keep the final score value in the range of 0 to 1. We applied the majority vote method before using the mapping function.</p>
      </sec>
      <sec>
        <title>Freshness</title>
        <p>Freshness represents the date of publication of a study, which is useful to consider to keep up with the advancement in a domain. We included this attribute in summarization with a higher rank assigned to more recent studies, following the less current studies, as described in <xref ref-type="table" rid="table2">Table 2</xref>.</p>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Assigned weights for research study year of publication.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="650"/>
            <col width="350"/>
            <thead>
              <tr valign="top">
                <td>Year of publication</td>
                <td>Rank</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Previous 1-5 years</td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>Previous 6-10 years</td>
                <td>2</td>
              </tr>
              <tr valign="top">
                <td>Previous 11-15 years</td>
                <td>3</td>
              </tr>
              <tr valign="top">
                <td>Other</td>
                <td>4</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
        <p>The final score was calculated according to Equation 5:</p>
        <disp-formula><italic>F<sub>score</sub></italic>=α(<italic>R<sub>i</sub></italic>→<italic>R<sub>f</sub></italic>) <bold>(5)</bold>,</disp-formula>
        <p>where <italic>R<sub>i</sub></italic> is the initial value assigned to each year obtained from <xref ref-type="table" rid="table2">Table 2</xref>, <italic>R<sub>f</sub></italic> is the mapped score obtained through mappings {1→10, 2→9, 3→8, and 4→7}, and α=0.1 is a scaling constant to keep the final score value in the range of 0 to 1.</p>
      </sec>
      <sec>
        <title>Text Selection for Summary</title>
        <p>The aggregate score calculated using Equation 1 provides the final rank values for a text sequence. Two types of summary structures can be generated: (1) a PICO-based summary, in which we select the top <italic>k</italic> text sequences in each part of PICO out of a total <italic>n</italic> number of sentences to be included in the final summary as &#124;<italic>n</italic>/<italic>k</italic>&#124; sentences from the Patient/Population component, &#124;<italic>n</italic>/<italic>k</italic>&#124; sentences from the Intervention component, and &#124;<italic>n</italic>/<italic>k</italic>&#124; from the Outcome component; and (2) a non-PICO-based summary, where we select the top <italic>k</italic> text sequences without considering their classification.</p>
      </sec>
      <sec>
        <title>Summary Presentation</title>
        <p>Our proposed model of automatic summarization can generate a summary for a single document or multiple documents simultaneously. For summaries of a single document, the selected set of representative PICO sequences is arranged in a template of Population/Problem, Intervention, and Outcome, as shown in <xref ref-type="boxed-text" rid="box2">Textbox 2</xref>. The templates are displayed according to the number of documents retained after a quality check. For instance, if we have a set of 5 studies retrieved against a user query, our algorithm will generate 5 individual summaries presented in the order of the most recent document summary followed by the others.</p>
        <boxed-text id="box2" position="float">
          <title>Example summary of a biomedical document represented with the Patient/Problem (P), Intervention (I), and Outcome (O) sequence.</title>
          <p>P: One hundred and fifty patients …duration at a frequency of at least once per week</p>
          <p>I: After patients underwent dural puncture … patients were positioned for surgery</p>
          <p>O: Number of follow-up appointments attended … occurrence of secondary ocular hypertension</p>
        </boxed-text>
        <p>In the case of a multidocument summary, our algorithm first identifies PICO sequences in all documents, finds the score for each sequence, selects the highest-scoring sequence in each category, and concatenates all scored sequences to obtain a combined summary of each type. In this case, the sequences included in the summary may or may not belong to a single document; however, each sequence is linked to its corresponding documents for audit, transparency, and education.</p>
      </sec>
      <sec>
        <title>Example Case: Step-by-Step Scenario of Summary Generation</title>
        <p>To clarify the steps of the proposed Biomed-Summarizer framework, we take the following example query: “How does family history affect rupture probability in intracranial aneurysms; is it a significant factor?“ (Abstract Query-Intracranial aneurysm family history). As shown in <xref rid="figure4" ref-type="fig">Figure 4</xref>, the user query was first abstracted from a detailed user query to increase the recall of retrieving studies. Second, the query was run on the PubMed search service, which returned a total of 239 studies, 130 of which were prognosis studies. Third, these studies were run through a quality model, “PQM,” which identified 74 studies as scientifically sound and the rest were filtered out. Fourth, the set of 74 studies was given to the PICO classification model in the form of text sequences, which were classified into five classes: Aim (32), Patients (9), Intervention (1), Results (168), and Outcome (49). Fifth, the PICO text sequences and the detailed query were passed through semantic similarity, in which the texts were first enriched semantically using medical ontologies, and the similarity score was calculated using the Jaccard coefficient. Sixth, the documents were ranked according to the accumulated score of four parameters: relevance score of the query and text, study type, venue’s credibility score, and freshness score. Finally, the required summary is created and presented to the user.</p>
        <fig id="figure4" position="float">
          <label>Figure 4</label>
          <caption>
            <p>Step-by-step scenario of query execution, retrieval of documents, quality checking, clinical context-aware (CCA) classification, semantic similarity, ranking, and summary creation. A: Aim; P: Population/Patients/Problem; I: Intervention; R: Results; O: Outcome; PICO: Patient/Problem, Intervention, Comparison, Outcome.</p>
          </caption>
          <graphic xlink:href="jmir_v22i10e19810_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Dataset Preparation</title>
        <p>To evaluate the performance of the proposed framework, we prepared two different datasets to test the proposed Biomed Summarizer framework. One dataset was used for the development and testing of the PQR model (D1), whereas the other was used for the development and testing of the CCA model (D2). The preparation protocols of both datasets are outlined in <xref ref-type="table" rid="table3">Table 3</xref>. These datasets are available to the research community via a GitHub link [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Dataset preparation protocols.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="200"/>
            <col width="400"/>
            <col width="400"/>
            <thead>
              <tr valign="top">
                <td>Preparation Protocol</td>
                <td>PQR<sup>a</sup> dataset (D1)</td>
                <td>CCA<sup>b</sup> dataset (D2)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Description</td>
                <td>This dataset was created for the quality assessment of biomedical studies related to the prognosis of brain aneurysm.</td>
                <td>This dataset was curated for the use of PICO<sup>c</sup> sequence classification. The final dataset was specific to the prognosis of brain aneurysm.</td>
              </tr>
              <tr valign="top">
                <td>Purpose</td>
                <td>To select only published documents that are scientifically rigorous for final summarization.</td>
                <td>To identify a sentence or a group of sentences for discovering the clinical context in terms of population, intervention, and outcomes.</td>
              </tr>
              <tr valign="top">
                <td>Methods</td>
                <td>The manual preparation of the dataset is a cumbersome job, and thus AI<sup>d</sup> models were used. For development of an AI model, a massive set of annotated documents is needed. Annotation is a tedious job; therefore, PubMed Clinical Queries (narrow) as a surrogate were used to obtain scientifically rigorous studies.</td>
                <td>N/A<sup>e</sup></td>
              </tr>
              <tr valign="top">
                <td>Data sources</td>
                <td>PubMed Database (for positive studies, the “Narrow[filter]” parameter was enabled).</td>
                <td>First, we collected a publicly available dataset, BioNLP 2018 [<xref ref-type="bibr" rid="ref21">21</xref>], which was classified based on the PICO sequence in addition to “Method” and “Results” elements. To increase the dataset size, we added more sentences related to brain aneurysm created from Medline abstracts retrieved using the NCBI<sup>f</sup> PubMed service Biopython Entrez library [<xref ref-type="bibr" rid="ref53">53</xref>].<break/>  <break/>  </td>
              </tr>
              <tr valign="top">
                <td>Query</td>
                <td>The term “(Prognosis/Narrow[filter]) AND (intracranial aneurysm)” was used as a query string.</td>
                <td>The term “Intracranial aneurysm” (along with its synonyms “cerebral aneurysm” and “brain aneurysm”) were used as a query string.</td>
              </tr>
              <tr valign="top">
                <td>Size</td>
                <td>2686 documents, including 697 positive (ie, scientifically rigorous) records</td>
                <td>A total of 173,000 PICO sequences (131,000 BioNLP+42,000 Brain Aneurysm) were included in the dataset.</td>
              </tr>
              <tr valign="top">
                <td>Inclusion/exclusion</td>
                <td>Only studies that were relevant and passed the criteria to be “Prognosis/Narrow[filter]” were included in the positive set. The other relevant studies not in the positive set were included in the negative set. All other studies were excluded from the final dataset.</td>
                <td>Only structured abstracts identified with at least one of the PICO elements were considered to extract the text sequence.</td>
              </tr>
              <tr valign="top">
                <td>Study types</td>
                <td>RCTs<sup>g</sup>, systematic reviews, and meta-analysis of RCTs were given more importance.</td>
                <td>RCTs, systematic reviews, and meta-analysis of RCTs were given more importance.</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table3fn1">
              <p><sup>a</sup>PQR: prognosis quality recognition.</p>
            </fn>
            <fn id="table3fn2">
              <p><sup>b</sup>CCA: clinical context–aware.</p>
            </fn>
            <fn id="table3fn3">
              <p><sup>c</sup>PICO: Patient/Problem, Intervention, Comparison, Outcome.</p>
            </fn>
            <fn id="table3fn4">
              <p><sup>d</sup>AI: artificial intelligence.</p>
            </fn>
            <fn id="table3fn5">
              <p><sup>e</sup>N/A: not applicable.</p>
            </fn>
            <fn id="table3fn6">
              <p><sup>f</sup>NCBI: National Center of Biotechnology Information.</p>
            </fn>
            <fn id="table3fn7">
              <p><sup>g</sup>RCT: randomized controlled trial.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Experimental Setup</title>
        <p>To evaluate the CCA and PQR classification models, we performed experiments on a 64-bit Windows operating system with an Intel Core i5 CPU, 3.20-Hz processor with 8-GB memory using dataset D1 and D2, respectively. The experiment was performed using RapidMiner studio [<xref ref-type="bibr" rid="ref54">54</xref>] to train and test the PQR models, whereas the python deep learning library Keras was used for the CCA classification model [<xref ref-type="bibr" rid="ref55">55</xref>].</p>
      </sec>
      <sec>
        <title>PQR Model</title>
        <p>The aim of this experiment was to quantify the comparative analysis of the proposed deep-learning model with other machine-learning models. Using the default settings of RapidMiner, the SVM model followed the kernel type “dot,” kernel cache of 200, and complexity constant <italic>c</italic> of 0.0; the DT criterion was “gain_ratio” with a maximum depth of 10; the <italic>k</italic> of the k-nearest neighbor was 5 with the mixed measure of “MixedEuclideanDistance.” The comparison of the performance of these models with that of a deep-learning model was assessed in terms of the F1-score, accuracy, and area under the curve (AUC) value, as shown in <xref ref-type="table" rid="table4">Table 4</xref>.</p>
        <p>We tuned the hyperparameters of the MLP by varying the hidden layers, size of layers, activation function, and epochs, and obtained varied results as shown in <xref ref-type="table" rid="table5">Table 5</xref>.</p>
        <table-wrap position="float" id="table4">
          <label>Table 4</label>
          <caption>
            <p>Comparative results of the deep-learning model with shallow machine-learning models.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="250"/>
            <col width="250"/>
            <col width="250"/>
            <col width="250"/>
            <thead>
              <tr valign="top">
                <td>Algorithm</td>
                <td>F1-score</td>
                <td>Accuracy</td>
                <td>AUC<sup>a</sup></td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Naïve Bayes</td>
                <td>90.83</td>
                <td>87.47</td>
                <td>0.987</td>
              </tr>
              <tr valign="top">
                <td>Decision tree</td>
                <td>85.10</td>
                <td>74.07</td>
                <td>0.50</td>
              </tr>
              <tr valign="top">
                <td>k-nearest neighbor</td>
                <td>46.53</td>
                <td>48.39</td>
                <td>0.829</td>
              </tr>
              <tr valign="top">
                <td>General linear model</td>
                <td>89.34</td>
                <td>82.38</td>
                <td>0.904</td>
              </tr>
              <tr valign="top">
                <td>Support vector machine</td>
                <td>86.96</td>
                <td>77.79</td>
                <td>0.983</td>
              </tr>
              <tr valign="top">
                <td>Deep learning (MLP<sup>b</sup>)</td>
                <td>93.17</td>
                <td>90.20</td>
                <td>0.967</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table4fn1">
              <p><sup>a</sup>AUC: area under the receiver operating characteristic curve.</p>
            </fn>
            <fn id="table4fn2">
              <p><sup>b</sup>MLP: multilayer perceptron.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table5">
          <label>Table 5</label>
          <caption>
            <p>Results of multilayer perceptron with varied hyperparameter settings.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="80"/>
            <col width="140"/>
            <col width="80"/>
            <col width="160"/>
            <col width="90"/>
            <col width="90"/>
            <col width="90"/>
            <col width="90"/>
            <col width="90"/>
            <col width="90"/>
            <thead>
              <tr valign="top">
                <td>Hidden layers (n)</td>
                <td>Hidden layer size</td>
                <td>BoW<sup>a</sup></td>
                <td>Activation</td>
                <td>Epochs (n)</td>
                <td>Recall</td>
                <td>Precision</td>
                <td>F1-score</td>
                <td>Accuracy</td>
                <td>AUC<sup>b</sup></td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>2</td>
                <td>50, 50</td>
                <td>No</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>90.28</td>
                <td>96.25</td>
                <td>93.17</td>
                <td>90.2</td>
                <td>0.967</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>No</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>93.47</td>
                <td>96.71</td>
                <td>95.06</td>
                <td>92.8</td>
                <td>0.969</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>No</td>
                <td>Maxout</td>
                <td>10</td>
                <td>96.82</td>
                <td>96.01</td>
                <td>96.41</td>
                <td>94.67</td>
                <td>0.976</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>No</td>
                <td>Maxout with Dropout</td>
                <td>10</td>
                <td>98.16</td>
                <td>93.61</td>
                <td>95.83</td>
                <td>93.67</td>
                <td>0.963</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>No</td>
                <td>Tanh</td>
                <td>10</td>
                <td>90.62</td>
                <td>97.65</td>
                <td>94.00</td>
                <td>91.44</td>
                <td>0.978</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>Yes</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>93.47</td>
                <td>98.24</td>
                <td>95.80</td>
                <td>93.92</td>
                <td>0.999</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>100, 50, 25</td>
                <td>Yes</td>
                <td>Maxout</td>
                <td>10</td>
                <td>94.47</td>
                <td>97.41</td>
                <td>95.92</td>
                <td>94.04</td>
                <td>0.977</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>50, 50, 50</td>
                <td>No</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>87.77</td>
                <td>96.86</td>
                <td>92.09</td>
                <td>88.83</td>
                <td>0.958</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>200, 100, 50</td>
                <td>No</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>92.96</td>
                <td>96.86</td>
                <td>94.87</td>
                <td>92.56</td>
                <td>0.975</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>200, 100, 50, 25</td>
                <td>No</td>
                <td>Rectifier</td>
                <td>10</td>
                <td>93.63</td>
                <td>96.05</td>
                <td>94.82</td>
                <td>92.43</td>
                <td>0.973</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table5fn1">
              <p><sup>a</sup>BoW: bag of words.</p>
            </fn>
            <fn id="table5fn2">
              <p><sup>b</sup>AUC: area under the receiver operating characteristic curve.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>As highlighted in <xref ref-type="table" rid="table5">Table 5</xref>, the highest accuracy and F1-score were obtained with the setting of 3 hidden layers consisting of 100, 50, and 25 neurons, respectively. The activation function was set to Maxout, and the model was trained on 10 epochs. Finally, we boosted the performance of the selected model with an ensemble approach using AdaBoost. The results of the optimized version of the proposed model are shown in <xref ref-type="table" rid="table6">Table 6</xref>, demonstrating an F1-score of about 97% and an accuracy of 95.41% with an AUC of 0.999.</p>
        <table-wrap position="float" id="table6">
          <label>Table 6</label>
          <caption>
            <p>Ensembling of deep-learning models.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="500"/>
            <col width="100"/>
            <col width="100"/>
            <col width="100"/>
            <col width="100"/>
            <col width="100"/>
            <thead>
              <tr valign="top">
                <td>Boosting Model</td>
                <td>Recall</td>
                <td>Precision</td>
                <td>F1-score</td>
                <td>Accuracy</td>
                <td>AUC<sup>a</sup></td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Ensemble voting (MLP<sup>b</sup>, DT<sup>c</sup>, NB<sup>d</sup>)</td>
                <td>95.81</td>
                <td>97.28</td>
                <td>96.54</td>
                <td>94.91</td>
                <td>0.955</td>
              </tr>
              <tr valign="top">
                <td>Proposed model (AdaBoost<sup>e</sup>-MLP)</td>
                <td>97.99</td>
                <td>95.9</td>
                <td>96.93</td>
                <td>95.41</td>
                <td>0.999</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table6fn1">
              <p><sup>a</sup>AUC: area under the receiver operating characteristic curve.</p>
            </fn>
            <fn id="table6fn2">
              <p><sup>b</sup>MLP: multilayer perceptron.</p>
            </fn>
            <fn id="table6fn3">
              <p><sup>c</sup>DT: decision tree.</p>
            </fn>
            <fn id="table6fn4">
              <p><sup>d</sup>NB: naïve Bayes.</p>
            </fn>
            <fn id="table6fn5">
              <p><sup>e</sup>AdaBoost: adaptive boosting.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>CCA Model</title>
        <p>We experimented with the combination of dataset D1 described above. The classification model results are shown in <xref ref-type="table" rid="table7">Table 7</xref>.</p>
        <p>The individual decision class performance of the proposed model is reported in <xref ref-type="table" rid="table8">Table 8</xref>. We found high F1-score values (≥80) for classes with top support (Aim, Outcome, Results, Population) as compared to the scores of the minor classes, which indicated that the dataset size needed to be increased for each of these minor classes to obtain a higher F1-score.</p>
        <table-wrap position="float" id="table7">
          <label>Table 7</label>
          <caption>
            <p>Comparative results of deep learning with traditional machine-learning models.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <thead>
              <tr valign="top">
                <td>Model</td>
                <td>Recall</td>
                <td>Precision</td>
                <td>F1-score</td>
                <td>Accuracy</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Logistic Regression</td>
                <td>0.42</td>
                <td>0.34</td>
                <td>0.36</td>
                <td>0.42</td>
              </tr>
              <tr valign="top">
                <td>AdaBoost<sup>a</sup></td>
                <td>0.49</td>
                <td>0.48</td>
                <td>0.46</td>
                <td>0.49</td>
              </tr>
              <tr valign="top">
                <td>Gradient Boost</td>
                <td>0.50</td>
                <td>0.59</td>
                <td>0.45</td>
                <td>0.50</td>
              </tr>
              <tr valign="top">
                <td>ANN<sup>b</sup></td>
                <td>0.29</td>
                <td>0.08</td>
                <td>0.13</td>
                <td>0.29</td>
              </tr>
              <tr valign="top">
                <td>kNN<sup>c</sup></td>
                <td>0.35</td>
                <td>0.36</td>
                <td>0.37</td>
                <td>0.35</td>
              </tr>
              <tr valign="top">
                <td>Proposed Bi-LSTM<sup>d</sup> model</td>
                <td>0.93</td>
                <td>0.94</td>
                <td>0.94</td>
                <td>0.93</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table7fn1">
              <p><sup>a</sup>AdaBoost: adaptive boosting.</p>
            </fn>
            <fn id="table7fn2">
              <p><sup>b</sup>ANN: artificial neural network.</p>
            </fn>
            <fn id="table7fn3">
              <p><sup>c</sup>KNN: k-nearest neighbor.</p>
            </fn>
            <fn id="table7fn4">
              <p><sup>d</sup>Bi-LSTM: bidirectional long-short term memory.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table8">
          <label>Table 8</label>
          <caption>
            <p>Precision, recall, F1-score, and support for individual classes of the proposed deep-learning model.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <col width="200"/>
            <thead>
              <tr valign="top">
                <td>Class</td>
                <td>Precision</td>
                <td>Recall</td>
                <td>F1-score</td>
                <td>Support</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Aim</td>
                <td>0.94</td>
                <td>0.95</td>
                <td>0.95</td>
                <td>3133</td>
              </tr>
              <tr valign="top">
                <td>Intervention</td>
                <td>0.84</td>
                <td>0.94</td>
                <td>0.89</td>
                <td>1238</td>
              </tr>
              <tr valign="top">
                <td>Outcome</td>
                <td>0.96</td>
                <td>0.94</td>
                <td>0.95</td>
                <td>5036</td>
              </tr>
              <tr valign="top">
                <td>Result</td>
                <td>0.96</td>
                <td>0.94</td>
                <td>0.95</td>
                <td>4852</td>
              </tr>
              <tr valign="top">
                <td>Population</td>
                <td>0.94</td>
                <td>0.95</td>
                <td>0.94</td>
                <td>3082</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Proposed Semantic Similarity Algorithm (JS<sup>2</sup>E)</title>
        <p>We measured the correlation strength based on the standard guideline of the Pearson correlation coefficient in the biomedical domain [<xref ref-type="bibr" rid="ref20">20</xref>]. We evaluated different similarity approaches, including string similarity (Jaccard), term frequency (count vectorizer), and pretrained word embedding using GloVe, Google word2vec, and fastText. The results showed that semantic enrichment is crucial to find the similarity between the texts because it significantly increases the size of the token set of texts by including synonyms, definitions, and prefLabel from ontologies. We tested this method on a well-known biomedical dataset, BIOSSES [<xref ref-type="bibr" rid="ref56">56</xref>], which contained 100 pairs of sentences manually annotated by the experts. As shown in <xref ref-type="table" rid="table9">Table 9</xref>, the Pearson correlation coefficient after semantic enrichment increased by 8.9% relative to that of the best-performing Jaccard similarity. Concerning the pretrained word-embedding model GloVe, word2vec (Google), and fastText (Facebook), the correlation improved by 41.8%, 60.5%, and 17.3%, respectively.</p>
        <table-wrap position="float" id="table9">
          <label>Table 9</label>
          <caption>
            <p>Comparison of similarity approaches.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="600"/>
            <col width="400"/>
            <thead>
              <tr valign="top">
                <td>Methods</td>
                <td>Pearson correlation coefficient (0.0-1.0)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Jaccard similarity</td>
                <td>0.56</td>
              </tr>
              <tr valign="top">
                <td>Cosine similarity (Count Vectorizer)</td>
                <td>0.54</td>
              </tr>
              <tr valign="top">
                <td>GloVe Embedding</td>
                <td>0.43</td>
              </tr>
              <tr valign="top">
                <td>Word2Vec (Google)</td>
                <td>0.38</td>
              </tr>
              <tr valign="top">
                <td>fastText (Facebook)</td>
                <td>0.52</td>
              </tr>
              <tr valign="top">
                <td>Jaccard Similarity after semantic enrichment (JS<sup>2</sup>E)</td>
                <td>0.61</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Expert Evaluation of Candidate and Reference Summaries</title>
        <p>The PICO-based summary obtained after classification and performing JS<sup>2</sup>E was then compared with the classical summary obtained using JS<sup>2</sup>E without classification. We considered the JS<sup>2</sup>E summary without classification as a baseline summary. Evaluation and comparison of two summaries were performed by three independent evaluators and scored between 0 and 5 on the following three metrics: summary relevance to the inbound query (M1); Aim, Population, Intervention, Results, and Outcome classification representation in the summary (M2); and model summary better than the baseline summary (M3).</p>
        <p><xref ref-type="table" rid="table10">Table 10</xref> shows the Pearson correlation coefficients of the scores of each evaluator (A, B, and C) concerning the average scores of the remaining two evaluators for each evaluation metric (M1, M2, and M3). There was a strong association among the scores of each evaluator concerning each metric, suggesting that automated summarization performed best on all three parameters. The lowest correlation coefficient was 0.728, which is still considered to be high on the correlation scale.</p>
        <p>The distribution of the scores by each of the evaluators concerning each metric is described in <xref ref-type="table" rid="table11">Table 11</xref>. The distribution suggests that there are sufficient instances for each of the scores in the evaluation dataset.</p>
        <table-wrap position="float" id="table10">
          <label>Table 10</label>
          <caption>
            <p>Correlation matrix (Pearson correlation coefficients) of similarity approaches among evaluators for summaries according to the three metrics.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="250"/>
            <col width="250"/>
            <col width="250"/>
            <col width="250"/>
            <thead>
              <tr valign="top">
                <td>Metric</td>
                <td>Evaluator A</td>
                <td>Evaluator B</td>
                <td>Evaluator C</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>M1<sup>a</sup></td>
                <td>0.728</td>
                <td>0.767</td>
                <td>0.837</td>
              </tr>
              <tr valign="top">
                <td>M2<sup>b</sup></td>
                <td>0.826</td>
                <td>0.924</td>
                <td>0.841</td>
              </tr>
              <tr valign="top">
                <td>M3<sup>c</sup></td>
                <td>0.772</td>
                <td>0.843</td>
                <td>0.804</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table10fn1">
              <p><sup>a</sup>M1: summary relevance to the inbound query.</p>
            </fn>
            <fn id="table10fn2">
              <p><sup>b</sup>M2: aim, population, intervention, results, and outcome classification representation in the summary.</p>
            </fn>
            <fn id="table10fn3">
              <p><sup>c</sup>M3: model summary better than the baseline summary.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table11">
          <label>Table 11</label>
          <caption>
            <p>Frequency distribution of scores with respect to each metric by all evaluators.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="340"/>
            <col width="320"/>
            <col width="340"/>
            <thead>
              <tr valign="top">
                <td>Metric</td>
                <td>Score</td>
                <td>Frequency</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>M1<sup>a</sup></td>
                <td>2</td>
                <td>4</td>
              </tr>
              <tr valign="top">
                <td>M1</td>
                <td>3</td>
                <td>10</td>
              </tr>
              <tr valign="top">
                <td>M1</td>
                <td>4</td>
                <td>12</td>
              </tr>
              <tr valign="top">
                <td>M1</td>
                <td>5</td>
                <td>4</td>
              </tr>
              <tr valign="top">
                <td>M2<sup>b</sup></td>
                <td>2</td>
                <td>3</td>
              </tr>
              <tr valign="top">
                <td>M2</td>
                <td>3</td>
                <td>10</td>
              </tr>
              <tr valign="top">
                <td>M2</td>
                <td>4</td>
                <td>10</td>
              </tr>
              <tr valign="top">
                <td>M2</td>
                <td>5</td>
                <td>7</td>
              </tr>
              <tr valign="top">
                <td>M3<sup>c</sup></td>
                <td>3</td>
                <td>12</td>
              </tr>
              <tr valign="top">
                <td>M3</td>
                <td>4</td>
                <td>16</td>
              </tr>
              <tr valign="top">
                <td>M3</td>
                <td>5</td>
                <td>2</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table11fn1">
              <p><sup>a</sup>M1: summary relevance to the inbound query.</p>
            </fn>
            <fn id="table11fn2">
              <p><sup>b</sup>M2: aim, population, intervention, results, and outcome classification representation in the summary.</p>
            </fn>
            <fn id="table11fn3">
              <p><sup>c</sup>M3: model summary better than the baseline summary.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>The proposed Biomed-Summarizer framework generates extractive summaries for single and multiple documents with acceptable accuracy. The evaluation results signify that the proposed framework performs significantly better than existing approaches, which was also evident from the correlation matrix generated by comparing the candidate and reference summaries obtained for each defined parameter. The PQR model trained on a large dataset of biomedical literature of intracranial aneurysm showed an accuracy of 95.41% in terms of recognizing quality articles. The CCA multiclass classification model outperformed the traditional machine-learning algorithms and achieved 93% accuracy for classifying five categories: Aim, Population, Intervention, Results, and Outcome. The semantic similarity algorithm demonstrated a significant Pearson correlation coefficient of 0.61 (on a 0-1 scale) from a well-known BIOSSES dataset after semantic enrichment, representing an improvement of 8.9% over the baseline Jaccard similarity score.</p>
        <p>An accurate clinical summarization is expected to revolutionize the field of domain-specific knowledge graphs [<xref ref-type="bibr" rid="ref57">57</xref>] and clinical decision support systems. For example, an individual PICO-extracted element from a biomedical text could be represented as a relationship in a knowledge graph, which could then be used for various clinical applications or could be directly supplied to clinical decision support systems for physicians to link to internal data-driven instances. The importance of linking external evidence (extracted from the biomedical literature) to internal evidence is important when the internal data are unable to capture all critical risk factors. One of our motivations was to extract the evidence from a PICO-based summarization of documents relevant to intracranial aneurysm (also known as cerebral or brain aneurysm). Since neither decision support systems nor knowledge graphs providing external evidence for complex neurological diseases such as intracranial aneurysm exist, there is a need to achieve automated summarization for such conditions to enhance translational research and clinical decision making synergistically. The proposed automated summarization framework will be pivotal to develop a hybrid machine-learning and knowledge-based clinical decision support system, termed NeuroAssist [<xref ref-type="bibr" rid="ref58">58</xref>], which aims to predict aneurysmal rupture (ie, subarachnoid hemorrhage). The other possible application areas of the proposed framework include automation of writing systematic reviews over a set of published clinical trials, extraction of evidence for evidence-based medicine, precision medicine, and development of clinical assistants/chatbots.</p>
        <p>The current work in the biomedical domain is mainly focused on issues of concept- or sentence-based relevance without relating a concept or sentence to a clinical context. Although the sentence-based classification approach is well-regarded for ATS, relying on only the relevance of a sentence without capturing its clinical context and semantics may lead to a clinically undesirable summary. Some of the summarization work focuses on creating summaries from abstracts only, which may result in low recall due to missing an important sequence of text that exists only in the body of the document. In addition, in previous works, the dataset used for training and testing [<xref ref-type="bibr" rid="ref29">29</xref>] contained only 1000 abstracts, which is not sufficient for a deep-learning model to be generalized. Recently, a new dataset was published in the BioNLP 2018 proceedings [<xref ref-type="bibr" rid="ref21">21</xref>], but it does not consider the quality evaluation of the source documents used for extracting the text for summarization. Therefore, we curated a dataset comprising over tens of thousands of abstracts from Medline and combined it with the BioNLP dataset.</p>
        <p>PICO-based ATS remains an unexplored area; however, work has been done on individual aspects such as PICO elements identification in the text [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref29">29</xref>,<xref ref-type="bibr" rid="ref30">30</xref>], quality recognition of therapy-related biomedical studies [<xref ref-type="bibr" rid="ref40">40</xref>,<xref ref-type="bibr" rid="ref41">41</xref>], and sentence-based summarization without PICO and quality evaluation [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref32">32</xref>]. To the best of our knowledge, the proposed Biomed-Summarizer framework for biomedical ATS is the first of its kind to use a quality recognition model, PICO-based classification using LSTM-based recurrent deep neural network model for key sentence identification trained on contextual vectors, and the JS<sup>2</sup>E algorithm for semantic matching of the user query and PICO text sequences.</p>
        <p>The proposed approach offers a few potential benefits compared to existing methods. First, unlike traditional machine-learning approaches that depend on features that are well-structured and less noisy, deep learning deals well with an unstructured noisy text. Therefore, our deep-learning model can be reused for domains using data of the same nature. Additionally, the accuracy of the proposed deep-learning model is expected to increase further if the volume of the dataset is extended. Second, our approach considers the quality evaluation of the documents being used in summarization in addition to publishing venue credibility, which offers two-fold benefits: it enhances the confidence of physicians on the system-generated summary and has applicability in real clinical settings, and, because it filters out the documents with the lowest quality from the retrieval set, the computational time of summary creation is reduced with respect to checking the similarity of the text with the query. Third, our proposed model is based on the PICO structure, which provides additional semantics as compared to a non-PICO approach in terms of understanding the problem, interventions, and outcomes.</p>
        <p>Traditional approaches are not cognizant of capturing the clinical context; therefore, the resultant summary includes sentences based on a high similarity score and sentence position, and is therefore less clinically effective. <xref ref-type="boxed-text" rid="box3">Textbox 3</xref> shows an example of a summary generated by the proposed method considering the clinical context; sequence 1 represents the aim of the study, sequence 2 represents the patient population included in the study, and sequence 3 represents the outcome of the study. In contrast, the conventional method selects the top three sequences based on high relevancy but misses the clinical context.</p>
        <boxed-text id="box3" position="float">
          <title>Effectiveness of Biomed-Summarizer in terms of clinical significance.</title>
          <p>
            <bold>Conventional Method</bold>
          </p>
          <disp-quote>
            <p>Sequence 1: The prevailing view is that larger aneurysms have a greater risk of rupture. Predicting the risk of aneurysmal rupture, especially for aneurysms with a relatively small diameter....</p>
          </disp-quote>
          <disp-quote>
            <p>Sequence 2: Alongside with the posterior communicating cerebral artery and middle cerebral artery of bifurcation, the anterior communicating cerebral artery (A com) is one ….</p>
          </disp-quote>
          <disp-quote>
            <p>Sequence 3: The A com artery is known to be one of the common sites of aneurysm rupture. Given the diversity of geometry, the difference of the dominance of A1 arteries ….</p>
          </disp-quote>
          <p>
            <bold>Proposed Method</bold>
          </p>
          <disp-quote>
            <p>Sequence 1: The aim of the present study was to evaluate whether the A com aneurysms behave differently from the aneurysms located elsewhere with respect to size being a rupture risk ….</p>
          </disp-quote>
          <disp-quote>
            <p>Sequence 2: Between January 2010 and December 2015, a total of 130 consecutive patients at our institution with the A com aneurysms-86 ruptured and 44 unruptured-were included in this study ….</p>
          </disp-quote>
          <disp-quote>
            <p>Sequence 3: The prevailing view is that larger aneurysms have a greater risk of rupture. Predicting the risk of aneurysmal rupture, especially for aneurysms with a relatively small diameter….</p>
          </disp-quote>
        </boxed-text>
      </sec>
      <sec>
        <title>Limitations and Future Directions</title>
        <p>The training datasets could be reviewed for the noise generated during the creation of automated annotation to obtain even more accurate results in PQR and CCA models. The current summary is a textual summary that contains sentences in the original form as they are presented in the source documents without any further processing to extract statistical information for an easy catch up of the contents.</p>
      </sec>
      <sec>
        <title>Conclusion</title>
        <p>Compared to traditional approaches, state-of-the-art deep neural network models can achieve high accuracy for an ATS task when trained on nonsparse semantically enriched features. Additionally, the automated pipeline for seeking research evidence can enable the seamless curation of voluminous biomedical literature to use in clinical decisions. By designing the proposed Biomed-Summarizer framework, we employed a set of methods for information extraction from credible published biomedical resources to construct a simplified summary that is precise, relevant, and contextually suited to clinical needs. We designed the framework to provide openness for other researchers to use, extend, or even make use of a subpart of it and extend for designing their own systems and services. Alongside the framework, we created a specialized dataset containing PICO elements and a few other text sequences such as Aim, Method, and Result for researchers to use in their experiments in the domain of brain aneurysm. The PICO dataset was extended using a custom data-mining process by incorporating the rigorous text processing techniques on PubMed research documents. This method can be further used to create a PICO dataset in other related biomedical domains by obtaining related research papers from PubMed. The evaluation results signify that the proposed Biomed-Summarizer framework performs significantly better than existing approaches.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Survey questionnaire.</p>
        <media xlink:href="jmir_v22i10e19810_app1.docx" xlink:title="DOCX File , 16 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AdaBoost</term>
          <def>
            <p>adaptive boosting</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">ATS</term>
          <def>
            <p>automatic text summarization</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">AUC</term>
          <def>
            <p>area under the receiver operating characteristic curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">Bi-LSTM</term>
          <def>
            <p>bidirectional long-short term memory</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">BoW</term>
          <def>
            <p>bag of words</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">CCA</term>
          <def>
            <p>clinical context–aware</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">JS<sup>2</sup>E</term>
          <def>
            <p>Jaccard similarity with semantic enrichments</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">MeSH</term>
          <def>
            <p>Medical Subject Heading</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">MLP</term>
          <def>
            <p>multilayer perceptron</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb10">PICO</term>
          <def>
            <p>Patient/Problem, Intervention, Comparison, and Outcome</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb11">PQR</term>
          <def>
            <p>prognosis quality recognition</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb12">RNN</term>
          <def>
            <p>recurrent neural network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb13">SVM</term>
          <def>
            <p>support vector machine</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This material is based upon work supported by the Brain Aneurysm Foundation and Oakland University, MI.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nasr Azadani</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ghadiri</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Davoodijam</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Graph-based biomedical text summarization: An itemset mining and sentence clustering approach</article-title>
          <source>J Biomed Inform</source>
          <year>2018</year>
          <month>08</month>
          <volume>84</volume>
          <fpage>42</fpage>
          <lpage>58</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(18)30111-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2018.06.005</pub-id>
          <pub-id pub-id-type="medline">29906584</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(18)30111-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schulze</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Neves</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Entity-Supported Summarization of Biomedical Abstracts</article-title>
          <source>The COLING 2016 Organizing Committee</source>
          <conf-name>Proceedings of the Fifth Workshop on Building and Evaluating Resources for Biomedical Text Mining (BioTxtM2016)</conf-name>
          <conf-date>December 2016</conf-date>
          <conf-loc>Osaka, Japan</conf-loc>
          <fpage>40</fpage>
          <lpage>49</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.aclweb.org/anthology/W16-5105/"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alami</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Meknassi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>En-nahnahi</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Enhancing unsupervised neural networks based text summarization with word embedding and ensemble learning</article-title>
          <source>Exp Syst App</source>
          <year>2019</year>
          <month>06</month>
          <volume>123</volume>
          <fpage>195</fpage>
          <lpage>211</lpage>
          <pub-id pub-id-type="doi">10.1016/j.eswa.2019.01.037</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Allahyari</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Pouriyeh</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Assefi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Safaei</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>D.</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>B.</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Kochut</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Text Summarization Techniques: A Brief Survey</article-title>
          <source>arXiv</source>
          <year>2017</year>
          <month>07</month>
          <day>28</day>
          <fpage>1</fpage>
          <lpage>9</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://arxiv.org/abs/1707.02268v3"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gambhir</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gupta</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Recent automatic text summarization techniques: a survey</article-title>
          <source>Artif Intell Rev</source>
          <year>2016</year>
          <month>3</month>
          <day>29</day>
          <volume>47</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>66</lpage>
          <pub-id pub-id-type="doi">10.1007/s10462-016-9475-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Khatri</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Parikh</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Abstractive and Extractive Text Summarization using Document Context Vector and Recurrent Neural Networks</article-title>
          <source>arxiv</source>
          <year>2018</year>
          <month>07</month>
          <day>29</day>
          <fpage>1807.08000</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1807.08000"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mishra</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Bian</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Fiszman</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Weir</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Jonnalagadda</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Mostafa</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Del Fiol</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Text summarization in the biomedical domain: a systematic review of recent research</article-title>
          <source>J Biomed Inform</source>
          <year>2014</year>
          <month>12</month>
          <volume>52</volume>
          <fpage>457</fpage>
          <lpage>467</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(14)00147-6"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2014.06.009</pub-id>
          <pub-id pub-id-type="medline">25016293</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(14)00147-6</pub-id>
          <pub-id pub-id-type="pmcid">PMC4261035</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afantenos</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Karkaletsis</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Stamatopoulos</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Summarization from medical documents: a survey</article-title>
          <source>Artif Intell Med</source>
          <year>2005</year>
          <month>02</month>
          <volume>33</volume>
          <issue>2</issue>
          <fpage>157</fpage>
          <lpage>177</lpage>
          <pub-id pub-id-type="doi">10.1016/j.artmed.2004.07.017</pub-id>
          <pub-id pub-id-type="medline">15811783</pub-id>
          <pub-id pub-id-type="pii">S0933-3657(04)00132-0</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bui</surname>
              <given-names>DDA</given-names>
            </name>
            <name name-style="western">
              <surname>Del Fiol</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Hurdle</surname>
              <given-names>JF</given-names>
            </name>
            <name name-style="western">
              <surname>Jonnalagadda</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Extractive text summarization system to aid data extraction from full text in systematic review development</article-title>
          <source>J Biomed Inform</source>
          <year>2016</year>
          <month>12</month>
          <volume>64</volume>
          <fpage>265</fpage>
          <lpage>272</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(16)30151-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2016.10.014</pub-id>
          <pub-id pub-id-type="medline">27989816</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(16)30151-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC5362293</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pivovarov</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Elhadad</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Automated methods for the summarization of electronic health records</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2015</year>
          <month>09</month>
          <volume>22</volume>
          <issue>5</issue>
          <fpage>938</fpage>
          <lpage>947</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/25882031"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/jamia/ocv032</pub-id>
          <pub-id pub-id-type="medline">25882031</pub-id>
          <pub-id pub-id-type="pii">ocv032</pub-id>
          <pub-id pub-id-type="pmcid">PMC4986665</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mikolov</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Yih</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Zweig</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Linguistic Regularities in Continuous Space Word Representations Internet</article-title>
          <year>2013</year>
          <month>06</month>
          <conf-name>Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies</conf-name>
          <conf-date>June 2013</conf-date>
          <conf-loc>Atlanta, Georgia</conf-loc>
          <publisher-name>Association for Computational Linguistics</publisher-name>
          <fpage>746</fpage>
          <lpage>751</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.aclweb.org/anthology/N13-1090/"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mikolov</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Corrado</surname>
              <given-names>GS</given-names>
            </name>
            <name name-style="western">
              <surname>Dean</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>Computing numeric representations of words in a high-dimensional space</article-title>
          <source>US Patent</source>
          <year>2015</year>
          <month>05</month>
          <day>19</day>
          <access-date>2019-08-05</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://patents.google.com/patent/US9740680B1/en">https://patents.google.com/patent/US9740680B1/en</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pennington</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Socher</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Manning</surname>
              <given-names>CD</given-names>
            </name>
          </person-group>
          <article-title>Glove: Global Vectors for Word Representation</article-title>
          <year>2014</year>
          <month>10</month>
          <conf-name>Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP)</conf-name>
          <conf-date>2014</conf-date>
          <conf-loc>Doha, Qatar</conf-loc>
          <publisher-name>Association for Computational Linguistics</publisher-name>
          <fpage>1532</fpage>
          <lpage>1543</lpage>
          <pub-id pub-id-type="doi">10.3115/v1/d14-1162</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Afzal</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Rastegar-Mojarad</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Kingsbury</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>A comparison of word embeddings for the biomedical natural language processing</article-title>
          <source>J Biomed Inform</source>
          <year>2018</year>
          <month>11</month>
          <volume>87</volume>
          <fpage>12</fpage>
          <lpage>20</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(18)30182-5"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2018.09.008</pub-id>
          <pub-id pub-id-type="medline">30217670</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(18)30182-5</pub-id>
          <pub-id pub-id-type="pmcid">PMC6585427</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Yin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Yan</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Mining E-commercial data: A text-rich heterogeneous network embedding approach</article-title>
          <year>2017</year>
          <month>05</month>
          <day>14</day>
          <conf-name>International Joint Conference on Neural Networks (IJCNN)</conf-name>
          <conf-date>May 14-19, 2017</conf-date>
          <conf-loc>Anchorage, AK</conf-loc>
          <publisher-name>IEEE</publisher-name>
          <fpage>1403</fpage>
          <lpage>1410</lpage>
          <pub-id pub-id-type="doi">10.1109/IJCNN.2017.7966017</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <article-title>Text data preprocessing</article-title>
          <source>Keras</source>
          <access-date>2020-10-07</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://keras.io/preprocessing/text/">https://keras.io/preprocessing/text/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sherstinsky</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Fundamentals of Recurrent Neural Network (RNN) and Long Short-Term Memory (LSTM) network</article-title>
          <source>Physica D: Nonlinear Phenomena</source>
          <year>2020</year>
          <month>03</month>
          <volume>404</volume>
          <fpage>132306</fpage>
          <pub-id pub-id-type="doi">10.1016/j.physd.2019.132306</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boudin</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Shi</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Nie</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Improving medical information retrieval with PICO element detection</article-title>
          <year>2010</year>
          <month>03</month>
          <conf-name>European Conference on Information Retrieval</conf-name>
          <conf-date>March 2010</conf-date>
          <conf-loc>Berlin, Heidelberg</conf-loc>
          <publisher-name>Springer</publisher-name>
          <pub-id pub-id-type="doi">10.1007/978-3-642-12275-0_8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>KC</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>CCH</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Xiao</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Liao</surname>
              <given-names>CC</given-names>
            </name>
            <name name-style="western">
              <surname>Chiang</surname>
              <given-names>IJ</given-names>
            </name>
          </person-group>
          <article-title>Classification of PICO elements by text features systematically extracted from PubMed abstracts</article-title>
          <year>2011</year>
          <month>11</month>
          <day>08</day>
          <conf-name>2011 IEEE International Conference on Granular Computing</conf-name>
          <conf-date>2011</conf-date>
          <conf-loc>Kaohsiung</conf-loc>
          <publisher-name>IEEE</publisher-name>
          <pub-id pub-id-type="doi">10.1109/grc.2011.6122608</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chabou</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Iglewski</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>PICO Extraction by combining the robustness of machine-learning methods with the rule-based methods</article-title>
          <year>2015</year>
          <month>06</month>
          <conf-name>2015 World Congr Inf Technol Comput Appl WCITCA 2015 Internet IEEE</conf-name>
          <conf-date>June 2015</conf-date>
          <conf-loc>Hammamet, Tunisia</conf-loc>
          <publisher-name>IEEE</publisher-name>
          <pub-id pub-id-type="doi">10.1109/wcitca.2015.7367038</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jin</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Szolovits</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>PICO Element Detection in Medical Text via Deep Neural Networks</article-title>
          <source>BioNLP 2018 Workshop</source>
          <year>2018</year>
          <month>07</month>
          <conf-name>Proceedings of the BioNLP 2018 workshop</conf-name>
          <conf-date>July 2018</conf-date>
          <conf-loc>Melbourne, Australia</conf-loc>
          <publisher-name>Association for Computational Linguistics</publisher-name>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.aclweb.org/anthology/papers/W/W18/W18-2308/"/>
          </comment>
          <pub-id pub-id-type="doi">10.18653/v1/w18-2308</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="web">
          <article-title>BioMed_Summarizer</article-title>
          <source>Brain_Aneurysm_Research</source>
          <access-date>2020-10-07</access-date>
          <publisher-name>GitHub</publisher-name>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://github.com/smileslab/Brain_Aneurysm_Research/tree/master/BioMed_Summarizer">https://github.com/smileslab/Brain_Aneurysm_Research/tree/master/BioMed_Summarizer</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Moradi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Small-world networks for summarization of biomedical articles</article-title>
          <source>arXiv</source>
          <year>2019</year>
          <month>03</month>
          <day>7</day>
          <fpage>1903.02861</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1903.02861"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Moradi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ghadiri</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Quantifying the informativeness for biomedical literature summarization: An itemset mining method</article-title>
          <source>Comput Methods Programs Biomed</source>
          <year>2017</year>
          <month>07</month>
          <volume>146</volume>
          <fpage>77</fpage>
          <lpage>89</lpage>
          <pub-id pub-id-type="doi">10.1016/j.cmpb.2017.05.011</pub-id>
          <pub-id pub-id-type="medline">28688492</pub-id>
          <pub-id pub-id-type="pii">S0169-2607(16)30925-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Luhn</surname>
              <given-names>HP</given-names>
            </name>
          </person-group>
          <article-title>The Automatic Creation of Literature Abstracts</article-title>
          <source>IBM J Res Dev</source>
          <year>1958</year>
          <month>4</month>
          <volume>2</volume>
          <issue>2</issue>
          <fpage>159</fpage>
          <lpage>165</lpage>
          <pub-id pub-id-type="doi">10.1147/rd.22.0159</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ferreira</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>de Souza Cabral</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Freitas</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Lins</surname>
              <given-names>RD</given-names>
            </name>
            <name name-style="western">
              <surname>de França Silva</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Simske</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Favaro</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>A multi-document summarization system based on statistics and linguistic treatment</article-title>
          <source>Exp Syst Appl</source>
          <year>2014</year>
          <month>10</month>
          <volume>41</volume>
          <issue>13</issue>
          <fpage>5780</fpage>
          <lpage>5787</lpage>
          <pub-id pub-id-type="doi">10.1016/j.eswa.2014.03.023</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lynn</surname>
              <given-names>HM</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>An improved method of automatic text summarization for web contents using lexical chain with semantic-related terms</article-title>
          <source>Soft Comput</source>
          <year>2017</year>
          <month>4</month>
          <day>27</day>
          <volume>22</volume>
          <issue>12</issue>
          <fpage>4013</fpage>
          <lpage>4023</lpage>
          <pub-id pub-id-type="doi">10.1007/s00500-017-2612-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Chiang</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Xiao</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Liao</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>PICO element detection in medical text without metadata: are first sentences enough?</article-title>
          <source>J Biomed Inform</source>
          <year>2013</year>
          <month>10</month>
          <volume>46</volume>
          <issue>5</issue>
          <fpage>940</fpage>
          <lpage>946</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(13)00110-X"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2013.07.009</pub-id>
          <pub-id pub-id-type="medline">23899909</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(13)00110-X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Martinez</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Cavedon</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Yencken</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Automatic classification of sentences to support Evidence Based Medicine</article-title>
          <source>BMC Bioinformatics</source>
          <year>2011</year>
          <volume>12</volume>
          <issue>Suppl 2</issue>
          <fpage>S5</fpage>
          <pub-id pub-id-type="doi">10.1186/1471-2105-12-s2-s5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Demner-Fushman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Answering Clinical Questions with Knowledge-Based and Statistical Techniques</article-title>
          <source>Comp Ling</source>
          <year>2007</year>
          <month>03</month>
          <volume>33</volume>
          <issue>1</issue>
          <fpage>63</fpage>
          <lpage>103</lpage>
          <pub-id pub-id-type="doi">10.1162/coli.2007.33.1.63</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zlabinger</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Hanbury</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Quasnik</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Brassey</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Medical entity corpus with pico elements and sentiment analysis</article-title>
          <source>European Language Resources Association (ELRA)</source>
          <year>2018</year>
          <month>05</month>
          <conf-name>Eleventh International Conference on Language Resources and Evaluation (LREC 2018)</conf-name>
          <conf-date>2018</conf-date>
          <conf-loc>Miyazaki, Japan</conf-loc>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wallace</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Kuiper</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sharma</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhu</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Marshall</surname>
              <given-names>I</given-names>
            </name>
          </person-group>
          <article-title>Extracting PICO Sentences from Clinical Trial Reports using Supervised Distant Supervision</article-title>
          <source>J Mach Learn Res</source>
          <year>2016</year>
          <volume>17</volume>
          <fpage>132</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/27746703"/>
          </comment>
          <pub-id pub-id-type="medline">27746703</pub-id>
          <pub-id pub-id-type="pmcid">PMC5065023</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boudin</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Nie</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bartlett</surname>
              <given-names>JC</given-names>
            </name>
            <name name-style="western">
              <surname>Grad</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Pluye</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Dawes</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Combining classifiers for robust PICO element detection</article-title>
          <source>BMC Med Inform Decis Mak</source>
          <year>2010</year>
          <month>05</month>
          <day>15</day>
          <volume>10</volume>
          <issue>1</issue>
          <fpage>29</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedinformdecismak.biomedcentral.com/articles/10.1186/1472-6947-10-29"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1472-6947-10-29</pub-id>
          <pub-id pub-id-type="medline">20470429</pub-id>
          <pub-id pub-id-type="pii">1472-6947-10-29</pub-id>
          <pub-id pub-id-type="pmcid">PMC2891622</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kilicoglu</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Demner-Fushman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Rindflesch</surname>
              <given-names>TC</given-names>
            </name>
            <name name-style="western">
              <surname>Wilczynski</surname>
              <given-names>NL</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>RB</given-names>
            </name>
          </person-group>
          <article-title>Towards automatic recognition of scientifically rigorous clinical research evidence</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2009</year>
          <volume>16</volume>
          <issue>1</issue>
          <fpage>25</fpage>
          <lpage>31</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/18952929"/>
          </comment>
          <pub-id pub-id-type="doi">10.1197/jamia.M2996</pub-id>
          <pub-id pub-id-type="medline">18952929</pub-id>
          <pub-id pub-id-type="pii">M2996</pub-id>
          <pub-id pub-id-type="pmcid">PMC2605595</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wilczynski</surname>
              <given-names>NL</given-names>
            </name>
            <name name-style="western">
              <surname>Morgan</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>RB</given-names>
            </name>
            <collab>Hedges Team</collab>
          </person-group>
          <article-title>An overview of the design and methods for retrieving high-quality studies for clinical care</article-title>
          <source>BMC Med Inform Decis Mak</source>
          <year>2005</year>
          <month>06</month>
          <day>21</day>
          <volume>5</volume>
          <fpage>20</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedinformdecismak.biomedcentral.com/articles/10.1186/1472-6947-5-20"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1472-6947-5-20</pub-id>
          <pub-id pub-id-type="medline">15969765</pub-id>
          <pub-id pub-id-type="pii">1472-6947-5-20</pub-id>
          <pub-id pub-id-type="pmcid">PMC1183213</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wilczynski</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>R</given-names>
            </name>
            <collab>Hedges Team</collab>
          </person-group>
          <article-title>Developing optimal search strategies for detecting clinically sound prognostic studies in MEDLINE: an analytic survey</article-title>
          <source>BMC Med</source>
          <year>2004</year>
          <month>06</month>
          <day>09</day>
          <volume>2</volume>
          <fpage>23</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedicine.biomedcentral.com/articles/10.1186/1741-7015-2-23"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1741-7015-2-23</pub-id>
          <pub-id pub-id-type="medline">15189561</pub-id>
          <pub-id pub-id-type="pii">1741-7015-2-23</pub-id>
          <pub-id pub-id-type="pmcid">PMC441418</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Aphinyanaphongs</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Tsamardinos</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Statnikov</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hardin</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Aliferis</surname>
              <given-names>CF</given-names>
            </name>
          </person-group>
          <article-title>Text categorization models for high-quality article retrieval in internal medicine</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2005</year>
          <volume>12</volume>
          <issue>2</issue>
          <fpage>207</fpage>
          <lpage>216</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/15561789"/>
          </comment>
          <pub-id pub-id-type="doi">10.1197/jamia.M1641</pub-id>
          <pub-id pub-id-type="medline">15561789</pub-id>
          <pub-id pub-id-type="pii">M1641</pub-id>
          <pub-id pub-id-type="pmcid">PMC551552</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Aphinyanaphongs</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Statnikov</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Aliferis</surname>
              <given-names>CF</given-names>
            </name>
          </person-group>
          <article-title>A comparison of citation metrics to machine learning filters for the identification of high quality MEDLINE documents</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2006</year>
          <volume>13</volume>
          <issue>4</issue>
          <fpage>446</fpage>
          <lpage>455</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/16622165"/>
          </comment>
          <pub-id pub-id-type="doi">10.1197/jamia.M2031</pub-id>
          <pub-id pub-id-type="medline">16622165</pub-id>
          <pub-id pub-id-type="pii">M2031</pub-id>
          <pub-id pub-id-type="pmcid">PMC1513679</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wilczynski</surname>
              <given-names>NL</given-names>
            </name>
            <name name-style="western">
              <surname>McKibbon</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Walter</surname>
              <given-names>SD</given-names>
            </name>
            <name name-style="western">
              <surname>Garg</surname>
              <given-names>AX</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>RB</given-names>
            </name>
          </person-group>
          <article-title>MEDLINE clinical queries are robust when searching in recent publishing years</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2013</year>
          <volume>20</volume>
          <issue>2</issue>
          <fpage>363</fpage>
          <lpage>368</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/23019242"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/amiajnl-2012-001075</pub-id>
          <pub-id pub-id-type="medline">23019242</pub-id>
          <pub-id pub-id-type="pii">amiajnl-2012-001075</pub-id>
          <pub-id pub-id-type="pmcid">PMC3638187</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Del Fiol</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Michelson</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Iorio</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Cotoi</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>A Deep Learning Method to Automatically Identify Reports of Scientifically Rigorous Clinical Research from the Biomedical Literature: Comparative Analytic Study</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>06</month>
          <day>25</day>
          <volume>20</volume>
          <issue>6</issue>
          <fpage>e10281</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/6/e10281/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10281</pub-id>
          <pub-id pub-id-type="medline">29941415</pub-id>
          <pub-id pub-id-type="pii">v20i6e10281</pub-id>
          <pub-id pub-id-type="pmcid">PMC6037944</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afzal</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hussain</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Malik</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Impact of Automatic Query Generation and Quality Recognition Using Deep Learning to Curate Evidence From Biomedical Literature: Empirical Study</article-title>
          <source>JMIR Med Inform</source>
          <year>2019</year>
          <month>12</month>
          <day>09</day>
          <volume>7</volume>
          <issue>4</issue>
          <fpage>e13430</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2019/4/e13430/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/13430</pub-id>
          <pub-id pub-id-type="medline">31815673</pub-id>
          <pub-id pub-id-type="pii">v7i4e13430</pub-id>
          <pub-id pub-id-type="pmcid">PMC6928703</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afzal</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hussain</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Haynes</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Context-aware grading of quality evidences for evidence-based decision-making</article-title>
          <source>Health Informatics J</source>
          <year>2019</year>
          <month>06</month>
          <volume>25</volume>
          <issue>2</issue>
          <fpage>429</fpage>
          <lpage>445</lpage>
          <pub-id pub-id-type="doi">10.1177/1460458217719560</pub-id>
          <pub-id pub-id-type="medline">28766402</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Irwin</surname>
              <given-names>AN</given-names>
            </name>
            <name name-style="western">
              <surname>Rackham</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Comparison of the time-to-indexing in PubMed between biomedical journals according to impact factor, discipline, and focus</article-title>
          <source>Res Social Adm Pharm</source>
          <year>2017</year>
          <volume>13</volume>
          <issue>2</issue>
          <fpage>389</fpage>
          <lpage>393</lpage>
          <pub-id pub-id-type="doi">10.1016/j.sapharm.2016.04.006</pub-id>
          <pub-id pub-id-type="medline">27215603</pub-id>
          <pub-id pub-id-type="pii">S1551-7411(16)30019-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Morid</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Fiszman</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Raja</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Jonnalagadda</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Del Fiol</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Classification of clinically useful sentences in clinical evidence resources</article-title>
          <source>J Biomed Inform</source>
          <year>2016</year>
          <month>04</month>
          <volume>60</volume>
          <fpage>14</fpage>
          <lpage>22</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(16)00004-6"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2016.01.003</pub-id>
          <pub-id pub-id-type="medline">26774763</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(16)00004-6</pub-id>
          <pub-id pub-id-type="pmcid">PMC4836984</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Goularte</surname>
              <given-names>FB</given-names>
            </name>
            <name name-style="western">
              <surname>Nassar</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Fileto</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Saggion</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>A text summarization method based on fuzzy rules and applicable to automated assessment</article-title>
          <source>Exp Syst Appl</source>
          <year>2019</year>
          <month>01</month>
          <volume>115</volume>
          <fpage>264</fpage>
          <lpage>275</lpage>
          <pub-id pub-id-type="doi">10.1016/j.eswa.2018.07.047</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cao</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wei</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>AttSum: Joint learning of focusing and summarization with neural attention</article-title>
          <source>arXiv</source>
          <year>2016</year>
          <month>09</month>
          <day>16</day>
          <fpage>1604.00125</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://arxiv.org/abs/1604.00125"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nallapati</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Zhai</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Zhou</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>SummaRuNNer: A Recurrent Neural Network Based Sequence Model for Extractive Summarization of Documents</article-title>
          <source>arxiv</source>
          <year>2016</year>
          <month>11</month>
          <day>14</day>
          <fpage>1611.04230</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://arxiv.org/abs/1611.04230"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cheng</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lapata</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Neural Summarization by Extracting Sentences and Words</article-title>
          <source>arXiv</source>
          <year>2016</year>
          <month>03</month>
          <day>23</day>
          <fpage>1603.07252</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1603.07252"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cao</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Wei</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Dong</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Zhou</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Ranking with Recursive Neural Networks and Its Application to Multi-Document Summarization</article-title>
          <year>2015</year>
          <month>01</month>
          <day>25</day>
          <conf-name>AAAI</conf-name>
          <conf-date>2015</conf-date>
          <conf-loc>Austin, Texas, USA</conf-loc>
          <fpage>2153</fpage>
          <lpage>2159</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Loper</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Bird</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>NLTK: The Natural Language Toolkit</article-title>
          <source>arXiv</source>
          <year>2002</year>
          <month>05</month>
          <day>17</day>
          <fpage>0205028</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/cs/0205028"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Atkins</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Briss</surname>
              <given-names>PA</given-names>
            </name>
            <name name-style="western">
              <surname>Eccles</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Flottorp</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Guyatt</surname>
              <given-names>GH</given-names>
            </name>
            <name name-style="western">
              <surname>Harbour</surname>
              <given-names>RT</given-names>
            </name>
            <name name-style="western">
              <surname>Hill</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Jaeschke</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Liberati</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Magrini</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Mason</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>O'Connell</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Oxman</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Phillips</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Schünemann</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Edejer</surname>
              <given-names>TT</given-names>
            </name>
            <name name-style="western">
              <surname>Vist</surname>
              <given-names>GE</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>JW</given-names>
            </name>
            <collab>GRADE Working Group</collab>
          </person-group>
          <article-title>Systems for grading the quality of evidence and the strength of recommendations II: pilot study of a new system</article-title>
          <source>BMC Health Serv Res</source>
          <year>2005</year>
          <month>03</month>
          <day>23</day>
          <volume>5</volume>
          <issue>1</issue>
          <fpage>25</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmchealthservres.biomedcentral.com/articles/10.1186/1472-6963-5-25"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1472-6963-5-25</pub-id>
          <pub-id pub-id-type="medline">15788089</pub-id>
          <pub-id pub-id-type="pii">1472-6963-5-25</pub-id>
          <pub-id pub-id-type="pmcid">PMC1084246</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kitto</surname>
              <given-names>SC</given-names>
            </name>
            <name name-style="western">
              <surname>Chesters</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Grbich</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Quality in qualitative research</article-title>
          <source>Med J Austral</source>
          <year>2008</year>
          <month>02</month>
          <day>18</day>
          <volume>188</volume>
          <issue>4</issue>
          <fpage>243</fpage>
          <lpage>246</lpage>
          <pub-id pub-id-type="doi">10.5694/j.1326-5377.2008.tb01595.x</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="web">
          <article-title>Bio.Entrez package</article-title>
          <source>Biopython</source>
          <access-date>2019-08-20</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://biopython.org/DIST/docs/api/Bio.Entrez-module.html">https://biopython.org/DIST/docs/api/Bio.Entrez-module.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="web">
          <source>RapidMiner</source>
          <access-date>2019-08-21</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://rapidminer.com/">https://rapidminer.com/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pedregosa</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Varoquaux</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Gramfort</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Michel</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Thirion</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Grisel</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Blondel</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Prettenhofer</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Weiss</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Dubourg</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Vanderplas</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Scikit-learn: Machine learning in Python</article-title>
          <source>J Machine Learn Res</source>
          <year>2011</year>
          <month>11</month>
          <fpage>2825</fpage>
          <lpage>2830</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmlr.org/papers/v12/pedregosa11a"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref56">
        <label>56</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sogancioglu</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Öztürk</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Özgür</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>BIOSSES: a semantic sentence similarity estimation system for the biomedical domain</article-title>
          <source>Bioinformatics</source>
          <year>2017</year>
          <month>07</month>
          <day>15</day>
          <volume>33</volume>
          <issue>14</issue>
          <fpage>i49</fpage>
          <lpage>i58</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/28881973"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/bioinformatics/btx238</pub-id>
          <pub-id pub-id-type="medline">28881973</pub-id>
          <pub-id pub-id-type="pii">3953954</pub-id>
          <pub-id pub-id-type="pmcid">PMC5870675</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref57">
        <label>57</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Malik</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Krishnamurthy</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Alobaidi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hussain</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Alam</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Malik</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Automated domain-specific healthcare knowledge graph curation framework: Subarachnoid hemorrhage as phenotype</article-title>
          <source>Exp Syst Appl</source>
          <year>2020</year>
          <month>05</month>
          <volume>145</volume>
          <fpage>113120</fpage>
          <pub-id pub-id-type="doi">10.1016/j.eswa.2019.113120</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref58">
        <label>58</label>
        <nlm-citation citation-type="web">
          <article-title>Neuroassist: an intelligent secure decision support system for prediction of brain aneurysm rupture, funded by Brain Aneurysm Foundation and Oakland University</article-title>
          <source>Smileslab</source>
          <year>2020</year>
          <access-date>2020-10-07</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.secs.oakland.edu/~Mahmood/details.html#one">https://www.secs.oakland.edu/~Mahmood/details.html#one</ext-link>
          </comment>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
